diff options
538 files changed, 12270 insertions, 7986 deletions
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8814e506f9..b754fbd662 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,7 +30,7 @@ By making a contribution to this project, I certify that: Erlang/otp is licensed under the Apache License 2.0 -As stated in: LICENSE.txt +As stated in: [LICENSE.txt](LICENSE.txt) http://developercertificate.org/ @@ -95,6 +95,9 @@ a discussion on the mailing list. * Make sure existing test cases don't fail. It is not necessary to run all tests (that would take many hours), but you should at least run the tests for the application you have changed. See [Running tests](https://github.com/erlang/otp/wiki/Running-tests). +* Make sure the documentation builds and is according to the dtd. eg. `make xmllint` or `cd lib/stdlib/ && make xmllint` +* Make sure no new dialyzer warnings have been added. eg. `make dialyzer` or `cd lib/stdlib/ && make dialyzer` +* Make sure that travis passes, if you go to https://travis-ci.org/$YOUR_GITHUB_USER/otp/ you can enable travis builds for you otp fork. Make sure that your branch contains clean commits: diff --git a/HOWTO/TESTING.md b/HOWTO/TESTING.md index ad59319efa..020be0309c 100644 --- a/HOWTO/TESTING.md +++ b/HOWTO/TESTING.md @@ -130,6 +130,52 @@ i.e. Running [ct_run][] from the command line still requires you to do the `ts:install()` step above. +### Convenience for running tests without the release and configuration steps + +It can be convenient to run tests with a single command. This way, one +do not need to worry about missing to run `make release_tests` after +changing a test suite. The `make test` command can be used for this +purpose. The `make test` command works when the current directory +contains a directory called test and in the root directory of the +source code tree. + +*(Waring)* Some test cases do not run correctly or cannot be run at +all through the `make test` command (typically test cases that require +test specific C code to be compiled) because `make test` runs tests +directly by invoking the `ct_run` command instead of using the `ts` +wrapper. One has to follow the procedure described above to run test +cases that do not work with `make test`. + +Below are some examples that illustrate how `make test` can be +used: + + # ERL_TOP needs to be set correctly + cd /path/to/otp + export ERL_TOP=`pwd` + + # Build Erlang/OTP + # + # Note that make test will only compile test code except when + # make test is executed from $ERL_TOP. + ./otp_build setup -a + + # Run a test case (The ARGS variable is passed to ct_run) + (cd $ERL_TOP/erts/emulator && make ARGS="-suite binary_SUITE -case deep_bitstr_lists" test) + + # Run a test suite + (cd $ERL_TOP/lib/stdlib && make ARGS="-suite ets_SUITE" test) + + # Run all test suites for an application + (cd $ERL_TOP/lib/asn1 && make test) + + # Run all tests + # + # When executed from $ERL_TOP, "make test" will first release and + # configure all tests and then attempt to run all tests with `ts:run`. + # This will take several hours. + (cd $ERL_TOP && make test) + + Examining the results --------------------- diff --git a/Makefile.in b/Makefile.in index 9be91a4b20..133ea0f4d6 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1148,3 +1148,11 @@ bootstrap_clean: || $(MAKE) BOOTSTRAP_ROOT=$(BOOTSTRAP_ROOT) bootstrap_root_clean # ---------------------------------------------------------------------- + +.PHONY: test dialyzer + +test: all release release_tests + $(ERL_TOP)/make/test_target_script.sh $(ERL_TOP) + +dialyzer: all + $(ERL_TOP)/scripts/run-dialyzer diff --git a/OTP_VERSION b/OTP_VERSION index 70a91e23ec..9854364f85 100644 --- a/OTP_VERSION +++ b/OTP_VERSION @@ -1 +1 @@ -22.1 +23.0-rc0 diff --git a/bootstrap/lib/compiler/ebin/beam_a.beam b/bootstrap/lib/compiler/ebin/beam_a.beam Binary files differindex ceb83ba48b..160855f0a1 100644 --- a/bootstrap/lib/compiler/ebin/beam_a.beam +++ b/bootstrap/lib/compiler/ebin/beam_a.beam diff --git a/bootstrap/lib/compiler/ebin/beam_asm.beam b/bootstrap/lib/compiler/ebin/beam_asm.beam Binary files differindex 78b75359a6..41b50928be 100644 --- a/bootstrap/lib/compiler/ebin/beam_asm.beam +++ b/bootstrap/lib/compiler/ebin/beam_asm.beam diff --git a/bootstrap/lib/compiler/ebin/beam_block.beam b/bootstrap/lib/compiler/ebin/beam_block.beam Binary files differindex 1db56cb5f1..0faa2a981c 100644 --- a/bootstrap/lib/compiler/ebin/beam_block.beam +++ b/bootstrap/lib/compiler/ebin/beam_block.beam diff --git a/bootstrap/lib/compiler/ebin/beam_call_types.beam b/bootstrap/lib/compiler/ebin/beam_call_types.beam Binary files differnew file mode 100644 index 0000000000..e73383b4ed --- /dev/null +++ b/bootstrap/lib/compiler/ebin/beam_call_types.beam diff --git a/bootstrap/lib/compiler/ebin/beam_clean.beam b/bootstrap/lib/compiler/ebin/beam_clean.beam Binary files differindex 4d567eca88..74a73324b9 100644 --- a/bootstrap/lib/compiler/ebin/beam_clean.beam +++ b/bootstrap/lib/compiler/ebin/beam_clean.beam diff --git a/bootstrap/lib/compiler/ebin/beam_dict.beam b/bootstrap/lib/compiler/ebin/beam_dict.beam Binary files differindex b9317a55ee..3dd6c8e6d7 100644 --- a/bootstrap/lib/compiler/ebin/beam_dict.beam +++ b/bootstrap/lib/compiler/ebin/beam_dict.beam diff --git a/bootstrap/lib/compiler/ebin/beam_disasm.beam b/bootstrap/lib/compiler/ebin/beam_disasm.beam Binary files differindex eb2a13d620..34619b82b9 100644 --- a/bootstrap/lib/compiler/ebin/beam_disasm.beam +++ b/bootstrap/lib/compiler/ebin/beam_disasm.beam diff --git a/bootstrap/lib/compiler/ebin/beam_except.beam b/bootstrap/lib/compiler/ebin/beam_except.beam Binary files differdeleted file mode 100644 index 1894483f71..0000000000 --- a/bootstrap/lib/compiler/ebin/beam_except.beam +++ /dev/null diff --git a/bootstrap/lib/compiler/ebin/beam_flatten.beam b/bootstrap/lib/compiler/ebin/beam_flatten.beam Binary files differindex d7780861c2..8874d5d6e8 100644 --- a/bootstrap/lib/compiler/ebin/beam_flatten.beam +++ b/bootstrap/lib/compiler/ebin/beam_flatten.beam diff --git a/bootstrap/lib/compiler/ebin/beam_jump.beam b/bootstrap/lib/compiler/ebin/beam_jump.beam Binary files differindex 289c08630a..3419693975 100644 --- a/bootstrap/lib/compiler/ebin/beam_jump.beam +++ b/bootstrap/lib/compiler/ebin/beam_jump.beam diff --git a/bootstrap/lib/compiler/ebin/beam_kernel_to_ssa.beam b/bootstrap/lib/compiler/ebin/beam_kernel_to_ssa.beam Binary files differindex 96362c8cdd..2d239b9e40 100644 --- a/bootstrap/lib/compiler/ebin/beam_kernel_to_ssa.beam +++ b/bootstrap/lib/compiler/ebin/beam_kernel_to_ssa.beam diff --git a/bootstrap/lib/compiler/ebin/beam_listing.beam b/bootstrap/lib/compiler/ebin/beam_listing.beam Binary files differindex 097ba5e7ff..747ee9e685 100644 --- a/bootstrap/lib/compiler/ebin/beam_listing.beam +++ b/bootstrap/lib/compiler/ebin/beam_listing.beam diff --git a/bootstrap/lib/compiler/ebin/beam_opcodes.beam b/bootstrap/lib/compiler/ebin/beam_opcodes.beam Binary files differindex 52a23f7f6c..31a700c6e9 100644 --- a/bootstrap/lib/compiler/ebin/beam_opcodes.beam +++ b/bootstrap/lib/compiler/ebin/beam_opcodes.beam diff --git a/bootstrap/lib/compiler/ebin/beam_peep.beam b/bootstrap/lib/compiler/ebin/beam_peep.beam Binary files differindex 9246bd1c27..293a6bb264 100644 --- a/bootstrap/lib/compiler/ebin/beam_peep.beam +++ b/bootstrap/lib/compiler/ebin/beam_peep.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa.beam b/bootstrap/lib/compiler/ebin/beam_ssa.beam Binary files differindex 0327230a58..7d59829998 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_bsm.beam b/bootstrap/lib/compiler/ebin/beam_ssa_bsm.beam Binary files differindex b9bb997cc7..9674c3c352 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_bsm.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_bsm.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_codegen.beam b/bootstrap/lib/compiler/ebin/beam_ssa_codegen.beam Binary files differindex cb06630012..7f2b8349fc 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_codegen.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_codegen.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_dead.beam b/bootstrap/lib/compiler/ebin/beam_ssa_dead.beam Binary files differindex 782073698e..3879c3c9d8 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_dead.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_dead.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_funs.beam b/bootstrap/lib/compiler/ebin/beam_ssa_funs.beam Binary files differindex 6370e1eb78..2efca72292 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_funs.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_funs.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_lint.beam b/bootstrap/lib/compiler/ebin/beam_ssa_lint.beam Binary files differindex e0a43057fc..d9f74f89dd 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_lint.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_lint.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_opt.beam b/bootstrap/lib/compiler/ebin/beam_ssa_opt.beam Binary files differindex b8588725b4..781820872b 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_opt.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_opt.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_pp.beam b/bootstrap/lib/compiler/ebin/beam_ssa_pp.beam Binary files differindex f76f15aa70..19bdaa1a9e 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_pp.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_pp.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_pre_codegen.beam b/bootstrap/lib/compiler/ebin/beam_ssa_pre_codegen.beam Binary files differindex e22cb5c846..2d8301e71b 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_pre_codegen.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_pre_codegen.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_recv.beam b/bootstrap/lib/compiler/ebin/beam_ssa_recv.beam Binary files differindex 578e8db0eb..d67d6db761 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_recv.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_recv.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_share.beam b/bootstrap/lib/compiler/ebin/beam_ssa_share.beam Binary files differindex d975cebf48..c59f105c65 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_share.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_share.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_type.beam b/bootstrap/lib/compiler/ebin/beam_ssa_type.beam Binary files differindex 0403230604..a52959ec5a 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_type.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_type.beam diff --git a/bootstrap/lib/compiler/ebin/beam_trim.beam b/bootstrap/lib/compiler/ebin/beam_trim.beam Binary files differindex 00f3224106..fefffbac73 100644 --- a/bootstrap/lib/compiler/ebin/beam_trim.beam +++ b/bootstrap/lib/compiler/ebin/beam_trim.beam diff --git a/bootstrap/lib/compiler/ebin/beam_types.beam b/bootstrap/lib/compiler/ebin/beam_types.beam Binary files differnew file mode 100644 index 0000000000..7658ed8dcd --- /dev/null +++ b/bootstrap/lib/compiler/ebin/beam_types.beam diff --git a/bootstrap/lib/compiler/ebin/beam_utils.beam b/bootstrap/lib/compiler/ebin/beam_utils.beam Binary files differindex 996525efc9..c26a840398 100644 --- a/bootstrap/lib/compiler/ebin/beam_utils.beam +++ b/bootstrap/lib/compiler/ebin/beam_utils.beam diff --git a/bootstrap/lib/compiler/ebin/beam_validator.beam b/bootstrap/lib/compiler/ebin/beam_validator.beam Binary files differindex 050789f481..b7eef824d1 100644 --- a/bootstrap/lib/compiler/ebin/beam_validator.beam +++ b/bootstrap/lib/compiler/ebin/beam_validator.beam diff --git a/bootstrap/lib/compiler/ebin/beam_z.beam b/bootstrap/lib/compiler/ebin/beam_z.beam Binary files differindex 21ca7bcf46..13a77364b8 100644 --- a/bootstrap/lib/compiler/ebin/beam_z.beam +++ b/bootstrap/lib/compiler/ebin/beam_z.beam diff --git a/bootstrap/lib/compiler/ebin/cerl.beam b/bootstrap/lib/compiler/ebin/cerl.beam Binary files differindex 948c4759b0..ef121c4d2f 100644 --- a/bootstrap/lib/compiler/ebin/cerl.beam +++ b/bootstrap/lib/compiler/ebin/cerl.beam diff --git a/bootstrap/lib/compiler/ebin/cerl_clauses.beam b/bootstrap/lib/compiler/ebin/cerl_clauses.beam Binary files differindex a20a4ca43b..09ce6fe0a8 100644 --- a/bootstrap/lib/compiler/ebin/cerl_clauses.beam +++ b/bootstrap/lib/compiler/ebin/cerl_clauses.beam diff --git a/bootstrap/lib/compiler/ebin/cerl_inline.beam b/bootstrap/lib/compiler/ebin/cerl_inline.beam Binary files differindex 7bf92a0dd7..e109cf60cc 100644 --- a/bootstrap/lib/compiler/ebin/cerl_inline.beam +++ b/bootstrap/lib/compiler/ebin/cerl_inline.beam diff --git a/bootstrap/lib/compiler/ebin/cerl_sets.beam b/bootstrap/lib/compiler/ebin/cerl_sets.beam Binary files differindex bda70130a6..58efd3f8e3 100644 --- a/bootstrap/lib/compiler/ebin/cerl_sets.beam +++ b/bootstrap/lib/compiler/ebin/cerl_sets.beam diff --git a/bootstrap/lib/compiler/ebin/cerl_trees.beam b/bootstrap/lib/compiler/ebin/cerl_trees.beam Binary files differindex 831eddc405..c7efc37edf 100644 --- a/bootstrap/lib/compiler/ebin/cerl_trees.beam +++ b/bootstrap/lib/compiler/ebin/cerl_trees.beam diff --git a/bootstrap/lib/compiler/ebin/compile.beam b/bootstrap/lib/compiler/ebin/compile.beam Binary files differindex ebe42547ee..64fe613410 100644 --- a/bootstrap/lib/compiler/ebin/compile.beam +++ b/bootstrap/lib/compiler/ebin/compile.beam diff --git a/bootstrap/lib/compiler/ebin/compiler.app b/bootstrap/lib/compiler/ebin/compiler.app index e086da1b68..4d6b9b484d 100644 --- a/bootstrap/lib/compiler/ebin/compiler.app +++ b/bootstrap/lib/compiler/ebin/compiler.app @@ -24,10 +24,10 @@ beam_a, beam_asm, beam_block, + beam_call_types, beam_clean, beam_dict, beam_disasm, - beam_except, beam_flatten, beam_jump, beam_kernel_to_ssa, @@ -47,6 +47,7 @@ beam_ssa_share, beam_ssa_type, beam_trim, + beam_types, beam_utils, beam_validator, beam_z, diff --git a/bootstrap/lib/compiler/ebin/core_lib.beam b/bootstrap/lib/compiler/ebin/core_lib.beam Binary files differindex 19b0c3dfcb..2179dceeca 100644 --- a/bootstrap/lib/compiler/ebin/core_lib.beam +++ b/bootstrap/lib/compiler/ebin/core_lib.beam diff --git a/bootstrap/lib/compiler/ebin/core_lint.beam b/bootstrap/lib/compiler/ebin/core_lint.beam Binary files differindex ecad4f33e0..76007eb4a3 100644 --- a/bootstrap/lib/compiler/ebin/core_lint.beam +++ b/bootstrap/lib/compiler/ebin/core_lint.beam diff --git a/bootstrap/lib/compiler/ebin/core_parse.beam b/bootstrap/lib/compiler/ebin/core_parse.beam Binary files differindex f14357cf02..10c3f135e0 100644 --- a/bootstrap/lib/compiler/ebin/core_parse.beam +++ b/bootstrap/lib/compiler/ebin/core_parse.beam diff --git a/bootstrap/lib/compiler/ebin/core_pp.beam b/bootstrap/lib/compiler/ebin/core_pp.beam Binary files differindex a2b3167e10..7ed91b8614 100644 --- a/bootstrap/lib/compiler/ebin/core_pp.beam +++ b/bootstrap/lib/compiler/ebin/core_pp.beam diff --git a/bootstrap/lib/compiler/ebin/core_scan.beam b/bootstrap/lib/compiler/ebin/core_scan.beam Binary files differindex 8e44464cd7..4505c07159 100644 --- a/bootstrap/lib/compiler/ebin/core_scan.beam +++ b/bootstrap/lib/compiler/ebin/core_scan.beam diff --git a/bootstrap/lib/compiler/ebin/erl_bifs.beam b/bootstrap/lib/compiler/ebin/erl_bifs.beam Binary files differindex e626e91f4f..32fc39a28b 100644 --- a/bootstrap/lib/compiler/ebin/erl_bifs.beam +++ b/bootstrap/lib/compiler/ebin/erl_bifs.beam diff --git a/bootstrap/lib/compiler/ebin/rec_env.beam b/bootstrap/lib/compiler/ebin/rec_env.beam Binary files differindex 6d0c5baa04..a3e441e12b 100644 --- a/bootstrap/lib/compiler/ebin/rec_env.beam +++ b/bootstrap/lib/compiler/ebin/rec_env.beam diff --git a/bootstrap/lib/compiler/ebin/sys_core_alias.beam b/bootstrap/lib/compiler/ebin/sys_core_alias.beam Binary files differindex b769b6be41..0cd375f706 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_alias.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_alias.beam diff --git a/bootstrap/lib/compiler/ebin/sys_core_bsm.beam b/bootstrap/lib/compiler/ebin/sys_core_bsm.beam Binary files differindex 6f4204e0a1..c92a95d2d6 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_bsm.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_bsm.beam diff --git a/bootstrap/lib/compiler/ebin/sys_core_fold.beam b/bootstrap/lib/compiler/ebin/sys_core_fold.beam Binary files differindex d4746e3664..e65b66aa99 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_fold.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_fold.beam diff --git a/bootstrap/lib/compiler/ebin/sys_core_fold_lists.beam b/bootstrap/lib/compiler/ebin/sys_core_fold_lists.beam Binary files differindex f4aac9ac13..fad3a34ad5 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_fold_lists.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_fold_lists.beam diff --git a/bootstrap/lib/compiler/ebin/sys_core_inline.beam b/bootstrap/lib/compiler/ebin/sys_core_inline.beam Binary files differindex 5647c8fcb5..a5c1fd8784 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_inline.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_inline.beam diff --git a/bootstrap/lib/compiler/ebin/sys_pre_attributes.beam b/bootstrap/lib/compiler/ebin/sys_pre_attributes.beam Binary files differindex 002b4f2914..96280ba9ec 100644 --- a/bootstrap/lib/compiler/ebin/sys_pre_attributes.beam +++ b/bootstrap/lib/compiler/ebin/sys_pre_attributes.beam diff --git a/bootstrap/lib/compiler/ebin/v3_core.beam b/bootstrap/lib/compiler/ebin/v3_core.beam Binary files differindex 3cbbdfd88b..214aacff2d 100644 --- a/bootstrap/lib/compiler/ebin/v3_core.beam +++ b/bootstrap/lib/compiler/ebin/v3_core.beam diff --git a/bootstrap/lib/compiler/ebin/v3_kernel.beam b/bootstrap/lib/compiler/ebin/v3_kernel.beam Binary files differindex 6ce6777e55..1a7bf1455b 100644 --- a/bootstrap/lib/compiler/ebin/v3_kernel.beam +++ b/bootstrap/lib/compiler/ebin/v3_kernel.beam diff --git a/bootstrap/lib/compiler/ebin/v3_kernel_pp.beam b/bootstrap/lib/compiler/ebin/v3_kernel_pp.beam Binary files differindex 5177b65198..4bbc799593 100644 --- a/bootstrap/lib/compiler/ebin/v3_kernel_pp.beam +++ b/bootstrap/lib/compiler/ebin/v3_kernel_pp.beam diff --git a/bootstrap/lib/kernel/ebin/application.beam b/bootstrap/lib/kernel/ebin/application.beam Binary files differindex 84c5b7636e..c72d5c645f 100644 --- a/bootstrap/lib/kernel/ebin/application.beam +++ b/bootstrap/lib/kernel/ebin/application.beam diff --git a/bootstrap/lib/kernel/ebin/application_controller.beam b/bootstrap/lib/kernel/ebin/application_controller.beam Binary files differindex b5d6545ca7..20a7077f9d 100644 --- a/bootstrap/lib/kernel/ebin/application_controller.beam +++ b/bootstrap/lib/kernel/ebin/application_controller.beam diff --git a/bootstrap/lib/kernel/ebin/application_master.beam b/bootstrap/lib/kernel/ebin/application_master.beam Binary files differindex 77bb9544e2..5a840ea9c6 100644 --- a/bootstrap/lib/kernel/ebin/application_master.beam +++ b/bootstrap/lib/kernel/ebin/application_master.beam diff --git a/bootstrap/lib/kernel/ebin/application_starter.beam b/bootstrap/lib/kernel/ebin/application_starter.beam Binary files differindex 0daab463ab..9abb2b8a0c 100644 --- a/bootstrap/lib/kernel/ebin/application_starter.beam +++ b/bootstrap/lib/kernel/ebin/application_starter.beam diff --git a/bootstrap/lib/kernel/ebin/auth.beam b/bootstrap/lib/kernel/ebin/auth.beam Binary files differindex c77178bf7c..724d74bcdc 100644 --- a/bootstrap/lib/kernel/ebin/auth.beam +++ b/bootstrap/lib/kernel/ebin/auth.beam diff --git a/bootstrap/lib/kernel/ebin/code.beam b/bootstrap/lib/kernel/ebin/code.beam Binary files differindex 204b15fcf2..2164f0d6fd 100644 --- a/bootstrap/lib/kernel/ebin/code.beam +++ b/bootstrap/lib/kernel/ebin/code.beam diff --git a/bootstrap/lib/kernel/ebin/code_server.beam b/bootstrap/lib/kernel/ebin/code_server.beam Binary files differindex 637e40f5d8..2c9e8c08ea 100644 --- a/bootstrap/lib/kernel/ebin/code_server.beam +++ b/bootstrap/lib/kernel/ebin/code_server.beam diff --git a/bootstrap/lib/kernel/ebin/disk_log.beam b/bootstrap/lib/kernel/ebin/disk_log.beam Binary files differindex 25142ee575..b17090c99a 100644 --- a/bootstrap/lib/kernel/ebin/disk_log.beam +++ b/bootstrap/lib/kernel/ebin/disk_log.beam diff --git a/bootstrap/lib/kernel/ebin/disk_log_1.beam b/bootstrap/lib/kernel/ebin/disk_log_1.beam Binary files differindex 0a781001dc..ee39281793 100644 --- a/bootstrap/lib/kernel/ebin/disk_log_1.beam +++ b/bootstrap/lib/kernel/ebin/disk_log_1.beam diff --git a/bootstrap/lib/kernel/ebin/disk_log_server.beam b/bootstrap/lib/kernel/ebin/disk_log_server.beam Binary files differindex 3b961dcefa..ca977a296e 100644 --- a/bootstrap/lib/kernel/ebin/disk_log_server.beam +++ b/bootstrap/lib/kernel/ebin/disk_log_server.beam diff --git a/bootstrap/lib/kernel/ebin/disk_log_sup.beam b/bootstrap/lib/kernel/ebin/disk_log_sup.beam Binary files differindex 67099f7212..58ca3f887f 100644 --- a/bootstrap/lib/kernel/ebin/disk_log_sup.beam +++ b/bootstrap/lib/kernel/ebin/disk_log_sup.beam diff --git a/bootstrap/lib/kernel/ebin/dist_ac.beam b/bootstrap/lib/kernel/ebin/dist_ac.beam Binary files differindex e82fc7dc75..77c3862368 100644 --- a/bootstrap/lib/kernel/ebin/dist_ac.beam +++ b/bootstrap/lib/kernel/ebin/dist_ac.beam diff --git a/bootstrap/lib/kernel/ebin/dist_util.beam b/bootstrap/lib/kernel/ebin/dist_util.beam Binary files differindex 8fa128dd23..7f764d68bb 100644 --- a/bootstrap/lib/kernel/ebin/dist_util.beam +++ b/bootstrap/lib/kernel/ebin/dist_util.beam diff --git a/bootstrap/lib/kernel/ebin/erl_boot_server.beam b/bootstrap/lib/kernel/ebin/erl_boot_server.beam Binary files differindex 801e3d8b88..5005621c01 100644 --- a/bootstrap/lib/kernel/ebin/erl_boot_server.beam +++ b/bootstrap/lib/kernel/ebin/erl_boot_server.beam diff --git a/bootstrap/lib/kernel/ebin/erl_compile_server.beam b/bootstrap/lib/kernel/ebin/erl_compile_server.beam Binary files differindex 28d79c2efb..43c77c4d7e 100644 --- a/bootstrap/lib/kernel/ebin/erl_compile_server.beam +++ b/bootstrap/lib/kernel/ebin/erl_compile_server.beam diff --git a/bootstrap/lib/kernel/ebin/erl_ddll.beam b/bootstrap/lib/kernel/ebin/erl_ddll.beam Binary files differindex c04bc43872..ec6900016a 100644 --- a/bootstrap/lib/kernel/ebin/erl_ddll.beam +++ b/bootstrap/lib/kernel/ebin/erl_ddll.beam diff --git a/bootstrap/lib/kernel/ebin/erl_distribution.beam b/bootstrap/lib/kernel/ebin/erl_distribution.beam Binary files differindex 4490f9c81f..71f01fbf30 100644 --- a/bootstrap/lib/kernel/ebin/erl_distribution.beam +++ b/bootstrap/lib/kernel/ebin/erl_distribution.beam diff --git a/bootstrap/lib/kernel/ebin/erl_epmd.beam b/bootstrap/lib/kernel/ebin/erl_epmd.beam Binary files differindex 6ed9862a0e..7d9d5cfba3 100644 --- a/bootstrap/lib/kernel/ebin/erl_epmd.beam +++ b/bootstrap/lib/kernel/ebin/erl_epmd.beam diff --git a/bootstrap/lib/kernel/ebin/erl_reply.beam b/bootstrap/lib/kernel/ebin/erl_reply.beam Binary files differindex 7658a35642..e74194895d 100644 --- a/bootstrap/lib/kernel/ebin/erl_reply.beam +++ b/bootstrap/lib/kernel/ebin/erl_reply.beam diff --git a/bootstrap/lib/kernel/ebin/erl_signal_handler.beam b/bootstrap/lib/kernel/ebin/erl_signal_handler.beam Binary files differindex da03769e2e..8876298e22 100644 --- a/bootstrap/lib/kernel/ebin/erl_signal_handler.beam +++ b/bootstrap/lib/kernel/ebin/erl_signal_handler.beam diff --git a/bootstrap/lib/kernel/ebin/error_handler.beam b/bootstrap/lib/kernel/ebin/error_handler.beam Binary files differindex 19739b0043..2f1c9075cc 100644 --- a/bootstrap/lib/kernel/ebin/error_handler.beam +++ b/bootstrap/lib/kernel/ebin/error_handler.beam diff --git a/bootstrap/lib/kernel/ebin/error_logger.beam b/bootstrap/lib/kernel/ebin/error_logger.beam Binary files differindex 10b13c35b6..a8694eedaa 100644 --- a/bootstrap/lib/kernel/ebin/error_logger.beam +++ b/bootstrap/lib/kernel/ebin/error_logger.beam diff --git a/bootstrap/lib/kernel/ebin/erts_debug.beam b/bootstrap/lib/kernel/ebin/erts_debug.beam Binary files differindex e0b759eee3..f731417e46 100644 --- a/bootstrap/lib/kernel/ebin/erts_debug.beam +++ b/bootstrap/lib/kernel/ebin/erts_debug.beam diff --git a/bootstrap/lib/kernel/ebin/file.beam b/bootstrap/lib/kernel/ebin/file.beam Binary files differindex a7a82a1f12..c151bcebbc 100644 --- a/bootstrap/lib/kernel/ebin/file.beam +++ b/bootstrap/lib/kernel/ebin/file.beam diff --git a/bootstrap/lib/kernel/ebin/file_io_server.beam b/bootstrap/lib/kernel/ebin/file_io_server.beam Binary files differindex b8f1272be4..1300423dc3 100644 --- a/bootstrap/lib/kernel/ebin/file_io_server.beam +++ b/bootstrap/lib/kernel/ebin/file_io_server.beam diff --git a/bootstrap/lib/kernel/ebin/file_server.beam b/bootstrap/lib/kernel/ebin/file_server.beam Binary files differindex cc2face79c..6f7ab15d43 100644 --- a/bootstrap/lib/kernel/ebin/file_server.beam +++ b/bootstrap/lib/kernel/ebin/file_server.beam diff --git a/bootstrap/lib/kernel/ebin/gen_sctp.beam b/bootstrap/lib/kernel/ebin/gen_sctp.beam Binary files differindex 9fc5800f0c..610f20716c 100644 --- a/bootstrap/lib/kernel/ebin/gen_sctp.beam +++ b/bootstrap/lib/kernel/ebin/gen_sctp.beam diff --git a/bootstrap/lib/kernel/ebin/gen_tcp.beam b/bootstrap/lib/kernel/ebin/gen_tcp.beam Binary files differindex 736b28c68f..af3ad4bb74 100644 --- a/bootstrap/lib/kernel/ebin/gen_tcp.beam +++ b/bootstrap/lib/kernel/ebin/gen_tcp.beam diff --git a/bootstrap/lib/kernel/ebin/gen_udp.beam b/bootstrap/lib/kernel/ebin/gen_udp.beam Binary files differindex 0d6541ee53..1c75188e26 100644 --- a/bootstrap/lib/kernel/ebin/gen_udp.beam +++ b/bootstrap/lib/kernel/ebin/gen_udp.beam diff --git a/bootstrap/lib/kernel/ebin/global.beam b/bootstrap/lib/kernel/ebin/global.beam Binary files differindex aadb5bf080..d4e39916f8 100644 --- a/bootstrap/lib/kernel/ebin/global.beam +++ b/bootstrap/lib/kernel/ebin/global.beam diff --git a/bootstrap/lib/kernel/ebin/global_group.beam b/bootstrap/lib/kernel/ebin/global_group.beam Binary files differindex 83885cdeb5..20c80de369 100644 --- a/bootstrap/lib/kernel/ebin/global_group.beam +++ b/bootstrap/lib/kernel/ebin/global_group.beam diff --git a/bootstrap/lib/kernel/ebin/global_search.beam b/bootstrap/lib/kernel/ebin/global_search.beam Binary files differindex a94d9da454..e56ccd0f53 100644 --- a/bootstrap/lib/kernel/ebin/global_search.beam +++ b/bootstrap/lib/kernel/ebin/global_search.beam diff --git a/bootstrap/lib/kernel/ebin/group.beam b/bootstrap/lib/kernel/ebin/group.beam Binary files differindex dcec55421c..63a28baf7a 100644 --- a/bootstrap/lib/kernel/ebin/group.beam +++ b/bootstrap/lib/kernel/ebin/group.beam diff --git a/bootstrap/lib/kernel/ebin/group_history.beam b/bootstrap/lib/kernel/ebin/group_history.beam Binary files differindex b9df7cfc14..456ed63c84 100644 --- a/bootstrap/lib/kernel/ebin/group_history.beam +++ b/bootstrap/lib/kernel/ebin/group_history.beam diff --git a/bootstrap/lib/kernel/ebin/heart.beam b/bootstrap/lib/kernel/ebin/heart.beam Binary files differindex 1b63142893..058290ee4e 100644 --- a/bootstrap/lib/kernel/ebin/heart.beam +++ b/bootstrap/lib/kernel/ebin/heart.beam diff --git a/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam b/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam Binary files differindex de7d9c2e57..35b9f544c0 100644 --- a/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam +++ b/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam diff --git a/bootstrap/lib/kernel/ebin/inet.beam b/bootstrap/lib/kernel/ebin/inet.beam Binary files differindex f64311e166..5ebc8d93fe 100644 --- a/bootstrap/lib/kernel/ebin/inet.beam +++ b/bootstrap/lib/kernel/ebin/inet.beam diff --git a/bootstrap/lib/kernel/ebin/inet6_sctp.beam b/bootstrap/lib/kernel/ebin/inet6_sctp.beam Binary files differindex a6220a9d69..496fc8d777 100644 --- a/bootstrap/lib/kernel/ebin/inet6_sctp.beam +++ b/bootstrap/lib/kernel/ebin/inet6_sctp.beam diff --git a/bootstrap/lib/kernel/ebin/inet6_tcp.beam b/bootstrap/lib/kernel/ebin/inet6_tcp.beam Binary files differindex 7cacde3399..0d98803e2d 100644 --- a/bootstrap/lib/kernel/ebin/inet6_tcp.beam +++ b/bootstrap/lib/kernel/ebin/inet6_tcp.beam diff --git a/bootstrap/lib/kernel/ebin/inet6_tcp_dist.beam b/bootstrap/lib/kernel/ebin/inet6_tcp_dist.beam Binary files differindex d1ca0b4f0d..aa53ec19db 100644 --- a/bootstrap/lib/kernel/ebin/inet6_tcp_dist.beam +++ b/bootstrap/lib/kernel/ebin/inet6_tcp_dist.beam diff --git a/bootstrap/lib/kernel/ebin/inet6_udp.beam b/bootstrap/lib/kernel/ebin/inet6_udp.beam Binary files differindex ac4be100e8..fb41f9f6ab 100644 --- a/bootstrap/lib/kernel/ebin/inet6_udp.beam +++ b/bootstrap/lib/kernel/ebin/inet6_udp.beam diff --git a/bootstrap/lib/kernel/ebin/inet_config.beam b/bootstrap/lib/kernel/ebin/inet_config.beam Binary files differindex c44b389b75..dde14ade38 100644 --- a/bootstrap/lib/kernel/ebin/inet_config.beam +++ b/bootstrap/lib/kernel/ebin/inet_config.beam diff --git a/bootstrap/lib/kernel/ebin/inet_db.beam b/bootstrap/lib/kernel/ebin/inet_db.beam Binary files differindex a835fa1a74..7b0604e3e8 100644 --- a/bootstrap/lib/kernel/ebin/inet_db.beam +++ b/bootstrap/lib/kernel/ebin/inet_db.beam diff --git a/bootstrap/lib/kernel/ebin/inet_dns.beam b/bootstrap/lib/kernel/ebin/inet_dns.beam Binary files differindex d88fa2185d..e2fb5beec5 100644 --- a/bootstrap/lib/kernel/ebin/inet_dns.beam +++ b/bootstrap/lib/kernel/ebin/inet_dns.beam diff --git a/bootstrap/lib/kernel/ebin/inet_gethost_native.beam b/bootstrap/lib/kernel/ebin/inet_gethost_native.beam Binary files differindex 8166eca5de..3d367d492e 100644 --- a/bootstrap/lib/kernel/ebin/inet_gethost_native.beam +++ b/bootstrap/lib/kernel/ebin/inet_gethost_native.beam diff --git a/bootstrap/lib/kernel/ebin/inet_hosts.beam b/bootstrap/lib/kernel/ebin/inet_hosts.beam Binary files differindex 6797eecb29..6435b84f69 100644 --- a/bootstrap/lib/kernel/ebin/inet_hosts.beam +++ b/bootstrap/lib/kernel/ebin/inet_hosts.beam diff --git a/bootstrap/lib/kernel/ebin/inet_parse.beam b/bootstrap/lib/kernel/ebin/inet_parse.beam Binary files differindex e8935134b5..7da09a44c9 100644 --- a/bootstrap/lib/kernel/ebin/inet_parse.beam +++ b/bootstrap/lib/kernel/ebin/inet_parse.beam diff --git a/bootstrap/lib/kernel/ebin/inet_res.beam b/bootstrap/lib/kernel/ebin/inet_res.beam Binary files differindex c964152d79..0fd60cb47e 100644 --- a/bootstrap/lib/kernel/ebin/inet_res.beam +++ b/bootstrap/lib/kernel/ebin/inet_res.beam diff --git a/bootstrap/lib/kernel/ebin/inet_sctp.beam b/bootstrap/lib/kernel/ebin/inet_sctp.beam Binary files differindex 0e659bd738..fecea26319 100644 --- a/bootstrap/lib/kernel/ebin/inet_sctp.beam +++ b/bootstrap/lib/kernel/ebin/inet_sctp.beam diff --git a/bootstrap/lib/kernel/ebin/inet_tcp.beam b/bootstrap/lib/kernel/ebin/inet_tcp.beam Binary files differindex c273d47012..f0eb540e4f 100644 --- a/bootstrap/lib/kernel/ebin/inet_tcp.beam +++ b/bootstrap/lib/kernel/ebin/inet_tcp.beam diff --git a/bootstrap/lib/kernel/ebin/inet_tcp_dist.beam b/bootstrap/lib/kernel/ebin/inet_tcp_dist.beam Binary files differindex d09aa250c8..b9a84e75c7 100644 --- a/bootstrap/lib/kernel/ebin/inet_tcp_dist.beam +++ b/bootstrap/lib/kernel/ebin/inet_tcp_dist.beam diff --git a/bootstrap/lib/kernel/ebin/inet_udp.beam b/bootstrap/lib/kernel/ebin/inet_udp.beam Binary files differindex 5853bde54f..e29d3e7c95 100644 --- a/bootstrap/lib/kernel/ebin/inet_udp.beam +++ b/bootstrap/lib/kernel/ebin/inet_udp.beam diff --git a/bootstrap/lib/kernel/ebin/kernel.beam b/bootstrap/lib/kernel/ebin/kernel.beam Binary files differindex 5b93fe805b..78c073ed5c 100644 --- a/bootstrap/lib/kernel/ebin/kernel.beam +++ b/bootstrap/lib/kernel/ebin/kernel.beam diff --git a/bootstrap/lib/kernel/ebin/kernel_config.beam b/bootstrap/lib/kernel/ebin/kernel_config.beam Binary files differindex e749cc6c96..ba45bb95d1 100644 --- a/bootstrap/lib/kernel/ebin/kernel_config.beam +++ b/bootstrap/lib/kernel/ebin/kernel_config.beam diff --git a/bootstrap/lib/kernel/ebin/kernel_refc.beam b/bootstrap/lib/kernel/ebin/kernel_refc.beam Binary files differindex 04306c6e2c..0e2db688ff 100644 --- a/bootstrap/lib/kernel/ebin/kernel_refc.beam +++ b/bootstrap/lib/kernel/ebin/kernel_refc.beam diff --git a/bootstrap/lib/kernel/ebin/local_tcp.beam b/bootstrap/lib/kernel/ebin/local_tcp.beam Binary files differindex 2958d19524..3df208b31f 100644 --- a/bootstrap/lib/kernel/ebin/local_tcp.beam +++ b/bootstrap/lib/kernel/ebin/local_tcp.beam diff --git a/bootstrap/lib/kernel/ebin/local_udp.beam b/bootstrap/lib/kernel/ebin/local_udp.beam Binary files differindex 33ed76e5cd..b1efe6c251 100644 --- a/bootstrap/lib/kernel/ebin/local_udp.beam +++ b/bootstrap/lib/kernel/ebin/local_udp.beam diff --git a/bootstrap/lib/kernel/ebin/logger.beam b/bootstrap/lib/kernel/ebin/logger.beam Binary files differindex 1dc1c4fef4..0da88b6003 100644 --- a/bootstrap/lib/kernel/ebin/logger.beam +++ b/bootstrap/lib/kernel/ebin/logger.beam diff --git a/bootstrap/lib/kernel/ebin/logger_backend.beam b/bootstrap/lib/kernel/ebin/logger_backend.beam Binary files differindex 40a3c4d80a..ddda136839 100644 --- a/bootstrap/lib/kernel/ebin/logger_backend.beam +++ b/bootstrap/lib/kernel/ebin/logger_backend.beam diff --git a/bootstrap/lib/kernel/ebin/logger_config.beam b/bootstrap/lib/kernel/ebin/logger_config.beam Binary files differindex 51c7f78237..21661ea33d 100644 --- a/bootstrap/lib/kernel/ebin/logger_config.beam +++ b/bootstrap/lib/kernel/ebin/logger_config.beam diff --git a/bootstrap/lib/kernel/ebin/logger_disk_log_h.beam b/bootstrap/lib/kernel/ebin/logger_disk_log_h.beam Binary files differindex dd0ca5f204..dafc128e2a 100644 --- a/bootstrap/lib/kernel/ebin/logger_disk_log_h.beam +++ b/bootstrap/lib/kernel/ebin/logger_disk_log_h.beam diff --git a/bootstrap/lib/kernel/ebin/logger_filters.beam b/bootstrap/lib/kernel/ebin/logger_filters.beam Binary files differindex 2d8035278d..c34c1efa4a 100644 --- a/bootstrap/lib/kernel/ebin/logger_filters.beam +++ b/bootstrap/lib/kernel/ebin/logger_filters.beam diff --git a/bootstrap/lib/kernel/ebin/logger_formatter.beam b/bootstrap/lib/kernel/ebin/logger_formatter.beam Binary files differindex e3e40460f0..ef791d2318 100644 --- a/bootstrap/lib/kernel/ebin/logger_formatter.beam +++ b/bootstrap/lib/kernel/ebin/logger_formatter.beam diff --git a/bootstrap/lib/kernel/ebin/logger_h_common.beam b/bootstrap/lib/kernel/ebin/logger_h_common.beam Binary files differindex 9e8c25b6b9..0e1a36663b 100644 --- a/bootstrap/lib/kernel/ebin/logger_h_common.beam +++ b/bootstrap/lib/kernel/ebin/logger_h_common.beam diff --git a/bootstrap/lib/kernel/ebin/logger_handler_watcher.beam b/bootstrap/lib/kernel/ebin/logger_handler_watcher.beam Binary files differindex fb20fd1818..7e5a439746 100644 --- a/bootstrap/lib/kernel/ebin/logger_handler_watcher.beam +++ b/bootstrap/lib/kernel/ebin/logger_handler_watcher.beam diff --git a/bootstrap/lib/kernel/ebin/logger_olp.beam b/bootstrap/lib/kernel/ebin/logger_olp.beam Binary files differindex ed06b6ae5e..4b21635585 100644 --- a/bootstrap/lib/kernel/ebin/logger_olp.beam +++ b/bootstrap/lib/kernel/ebin/logger_olp.beam diff --git a/bootstrap/lib/kernel/ebin/logger_proxy.beam b/bootstrap/lib/kernel/ebin/logger_proxy.beam Binary files differindex ed0ed0f56b..7f97031ab8 100644 --- a/bootstrap/lib/kernel/ebin/logger_proxy.beam +++ b/bootstrap/lib/kernel/ebin/logger_proxy.beam diff --git a/bootstrap/lib/kernel/ebin/logger_server.beam b/bootstrap/lib/kernel/ebin/logger_server.beam Binary files differindex dcff9beac7..069cf723e6 100644 --- a/bootstrap/lib/kernel/ebin/logger_server.beam +++ b/bootstrap/lib/kernel/ebin/logger_server.beam diff --git a/bootstrap/lib/kernel/ebin/logger_simple_h.beam b/bootstrap/lib/kernel/ebin/logger_simple_h.beam Binary files differindex 583c77b290..9ccd980af7 100644 --- a/bootstrap/lib/kernel/ebin/logger_simple_h.beam +++ b/bootstrap/lib/kernel/ebin/logger_simple_h.beam diff --git a/bootstrap/lib/kernel/ebin/logger_std_h.beam b/bootstrap/lib/kernel/ebin/logger_std_h.beam Binary files differindex c8eb73a076..614888589b 100644 --- a/bootstrap/lib/kernel/ebin/logger_std_h.beam +++ b/bootstrap/lib/kernel/ebin/logger_std_h.beam diff --git a/bootstrap/lib/kernel/ebin/logger_sup.beam b/bootstrap/lib/kernel/ebin/logger_sup.beam Binary files differindex 60192ef4a1..98d52f7e93 100644 --- a/bootstrap/lib/kernel/ebin/logger_sup.beam +++ b/bootstrap/lib/kernel/ebin/logger_sup.beam diff --git a/bootstrap/lib/kernel/ebin/net.beam b/bootstrap/lib/kernel/ebin/net.beam Binary files differindex 44b1f10603..a1c8d8b526 100644 --- a/bootstrap/lib/kernel/ebin/net.beam +++ b/bootstrap/lib/kernel/ebin/net.beam diff --git a/bootstrap/lib/kernel/ebin/net_adm.beam b/bootstrap/lib/kernel/ebin/net_adm.beam Binary files differindex 5810577c6a..4fb48aba84 100644 --- a/bootstrap/lib/kernel/ebin/net_adm.beam +++ b/bootstrap/lib/kernel/ebin/net_adm.beam diff --git a/bootstrap/lib/kernel/ebin/net_kernel.beam b/bootstrap/lib/kernel/ebin/net_kernel.beam Binary files differindex 4c57f5f6c8..40c6ea85bd 100644 --- a/bootstrap/lib/kernel/ebin/net_kernel.beam +++ b/bootstrap/lib/kernel/ebin/net_kernel.beam diff --git a/bootstrap/lib/kernel/ebin/os.beam b/bootstrap/lib/kernel/ebin/os.beam Binary files differindex f05007041b..41a45ca812 100644 --- a/bootstrap/lib/kernel/ebin/os.beam +++ b/bootstrap/lib/kernel/ebin/os.beam diff --git a/bootstrap/lib/kernel/ebin/pg2.beam b/bootstrap/lib/kernel/ebin/pg2.beam Binary files differindex d293ea18f3..ce1c5270a9 100644 --- a/bootstrap/lib/kernel/ebin/pg2.beam +++ b/bootstrap/lib/kernel/ebin/pg2.beam diff --git a/bootstrap/lib/kernel/ebin/ram_file.beam b/bootstrap/lib/kernel/ebin/ram_file.beam Binary files differindex 02c4c6454a..e00c4d3d34 100644 --- a/bootstrap/lib/kernel/ebin/ram_file.beam +++ b/bootstrap/lib/kernel/ebin/ram_file.beam diff --git a/bootstrap/lib/kernel/ebin/raw_file_io.beam b/bootstrap/lib/kernel/ebin/raw_file_io.beam Binary files differindex 16dc314ba5..03c5935a42 100644 --- a/bootstrap/lib/kernel/ebin/raw_file_io.beam +++ b/bootstrap/lib/kernel/ebin/raw_file_io.beam diff --git a/bootstrap/lib/kernel/ebin/raw_file_io_compressed.beam b/bootstrap/lib/kernel/ebin/raw_file_io_compressed.beam Binary files differindex 3c41f8528e..c3f9da47dc 100644 --- a/bootstrap/lib/kernel/ebin/raw_file_io_compressed.beam +++ b/bootstrap/lib/kernel/ebin/raw_file_io_compressed.beam diff --git a/bootstrap/lib/kernel/ebin/raw_file_io_deflate.beam b/bootstrap/lib/kernel/ebin/raw_file_io_deflate.beam Binary files differindex 11b8f4a6db..dd095a4b0b 100644 --- a/bootstrap/lib/kernel/ebin/raw_file_io_deflate.beam +++ b/bootstrap/lib/kernel/ebin/raw_file_io_deflate.beam diff --git a/bootstrap/lib/kernel/ebin/raw_file_io_delayed.beam b/bootstrap/lib/kernel/ebin/raw_file_io_delayed.beam Binary files differindex b3381cc397..97f7023c1d 100644 --- a/bootstrap/lib/kernel/ebin/raw_file_io_delayed.beam +++ b/bootstrap/lib/kernel/ebin/raw_file_io_delayed.beam diff --git a/bootstrap/lib/kernel/ebin/raw_file_io_inflate.beam b/bootstrap/lib/kernel/ebin/raw_file_io_inflate.beam Binary files differindex 7c47f2cf10..6bc4a1e7a0 100644 --- a/bootstrap/lib/kernel/ebin/raw_file_io_inflate.beam +++ b/bootstrap/lib/kernel/ebin/raw_file_io_inflate.beam diff --git a/bootstrap/lib/kernel/ebin/raw_file_io_list.beam b/bootstrap/lib/kernel/ebin/raw_file_io_list.beam Binary files differindex b544fd394e..eeafb97ea5 100644 --- a/bootstrap/lib/kernel/ebin/raw_file_io_list.beam +++ b/bootstrap/lib/kernel/ebin/raw_file_io_list.beam diff --git a/bootstrap/lib/kernel/ebin/raw_file_io_raw.beam b/bootstrap/lib/kernel/ebin/raw_file_io_raw.beam Binary files differindex b47a67055d..fab63a6629 100644 --- a/bootstrap/lib/kernel/ebin/raw_file_io_raw.beam +++ b/bootstrap/lib/kernel/ebin/raw_file_io_raw.beam diff --git a/bootstrap/lib/kernel/ebin/rpc.beam b/bootstrap/lib/kernel/ebin/rpc.beam Binary files differindex b53525e2ca..493223db2f 100644 --- a/bootstrap/lib/kernel/ebin/rpc.beam +++ b/bootstrap/lib/kernel/ebin/rpc.beam diff --git a/bootstrap/lib/kernel/ebin/seq_trace.beam b/bootstrap/lib/kernel/ebin/seq_trace.beam Binary files differindex f81a39ddc0..fdd399092e 100644 --- a/bootstrap/lib/kernel/ebin/seq_trace.beam +++ b/bootstrap/lib/kernel/ebin/seq_trace.beam diff --git a/bootstrap/lib/kernel/ebin/standard_error.beam b/bootstrap/lib/kernel/ebin/standard_error.beam Binary files differindex 18582d22f2..96dc4d5433 100644 --- a/bootstrap/lib/kernel/ebin/standard_error.beam +++ b/bootstrap/lib/kernel/ebin/standard_error.beam diff --git a/bootstrap/lib/kernel/ebin/user.beam b/bootstrap/lib/kernel/ebin/user.beam Binary files differindex 8214d9d6aa..59a7cdaf6d 100644 --- a/bootstrap/lib/kernel/ebin/user.beam +++ b/bootstrap/lib/kernel/ebin/user.beam diff --git a/bootstrap/lib/kernel/ebin/user_drv.beam b/bootstrap/lib/kernel/ebin/user_drv.beam Binary files differindex 43df5afa36..0dab31f7d1 100644 --- a/bootstrap/lib/kernel/ebin/user_drv.beam +++ b/bootstrap/lib/kernel/ebin/user_drv.beam diff --git a/bootstrap/lib/kernel/ebin/user_sup.beam b/bootstrap/lib/kernel/ebin/user_sup.beam Binary files differindex c82fca70da..68f3a330c2 100644 --- a/bootstrap/lib/kernel/ebin/user_sup.beam +++ b/bootstrap/lib/kernel/ebin/user_sup.beam diff --git a/bootstrap/lib/kernel/ebin/wrap_log_reader.beam b/bootstrap/lib/kernel/ebin/wrap_log_reader.beam Binary files differindex ceb8dd9a78..7f719ee8a5 100644 --- a/bootstrap/lib/kernel/ebin/wrap_log_reader.beam +++ b/bootstrap/lib/kernel/ebin/wrap_log_reader.beam diff --git a/bootstrap/lib/stdlib/ebin/array.beam b/bootstrap/lib/stdlib/ebin/array.beam Binary files differindex 9e783452f5..c0a14da511 100644 --- a/bootstrap/lib/stdlib/ebin/array.beam +++ b/bootstrap/lib/stdlib/ebin/array.beam diff --git a/bootstrap/lib/stdlib/ebin/base64.beam b/bootstrap/lib/stdlib/ebin/base64.beam Binary files differindex 5872970da7..246bbc8fea 100644 --- a/bootstrap/lib/stdlib/ebin/base64.beam +++ b/bootstrap/lib/stdlib/ebin/base64.beam diff --git a/bootstrap/lib/stdlib/ebin/beam_lib.beam b/bootstrap/lib/stdlib/ebin/beam_lib.beam Binary files differindex a96ccb6f8c..e815fc298e 100644 --- a/bootstrap/lib/stdlib/ebin/beam_lib.beam +++ b/bootstrap/lib/stdlib/ebin/beam_lib.beam diff --git a/bootstrap/lib/stdlib/ebin/binary.beam b/bootstrap/lib/stdlib/ebin/binary.beam Binary files differindex fc1bbe163a..0712d0c64b 100644 --- a/bootstrap/lib/stdlib/ebin/binary.beam +++ b/bootstrap/lib/stdlib/ebin/binary.beam diff --git a/bootstrap/lib/stdlib/ebin/c.beam b/bootstrap/lib/stdlib/ebin/c.beam Binary files differindex 17e061b1d0..6cea2d1e9d 100644 --- a/bootstrap/lib/stdlib/ebin/c.beam +++ b/bootstrap/lib/stdlib/ebin/c.beam diff --git a/bootstrap/lib/stdlib/ebin/calendar.beam b/bootstrap/lib/stdlib/ebin/calendar.beam Binary files differindex 89102adda8..6c5773d852 100644 --- a/bootstrap/lib/stdlib/ebin/calendar.beam +++ b/bootstrap/lib/stdlib/ebin/calendar.beam diff --git a/bootstrap/lib/stdlib/ebin/dets.beam b/bootstrap/lib/stdlib/ebin/dets.beam Binary files differindex 2eadde21ff..4af2c67a93 100644 --- a/bootstrap/lib/stdlib/ebin/dets.beam +++ b/bootstrap/lib/stdlib/ebin/dets.beam diff --git a/bootstrap/lib/stdlib/ebin/dets_server.beam b/bootstrap/lib/stdlib/ebin/dets_server.beam Binary files differindex 8cd247e3be..ee1b586ef2 100644 --- a/bootstrap/lib/stdlib/ebin/dets_server.beam +++ b/bootstrap/lib/stdlib/ebin/dets_server.beam diff --git a/bootstrap/lib/stdlib/ebin/dets_sup.beam b/bootstrap/lib/stdlib/ebin/dets_sup.beam Binary files differindex 0b9fb6379f..b198d1f872 100644 --- a/bootstrap/lib/stdlib/ebin/dets_sup.beam +++ b/bootstrap/lib/stdlib/ebin/dets_sup.beam diff --git a/bootstrap/lib/stdlib/ebin/dets_utils.beam b/bootstrap/lib/stdlib/ebin/dets_utils.beam Binary files differindex e38eb384a1..31eeb4375c 100644 --- a/bootstrap/lib/stdlib/ebin/dets_utils.beam +++ b/bootstrap/lib/stdlib/ebin/dets_utils.beam diff --git a/bootstrap/lib/stdlib/ebin/dets_v9.beam b/bootstrap/lib/stdlib/ebin/dets_v9.beam Binary files differindex f73509ae3f..67abb787dc 100644 --- a/bootstrap/lib/stdlib/ebin/dets_v9.beam +++ b/bootstrap/lib/stdlib/ebin/dets_v9.beam diff --git a/bootstrap/lib/stdlib/ebin/dict.beam b/bootstrap/lib/stdlib/ebin/dict.beam Binary files differindex dc390024a4..c3155b9129 100644 --- a/bootstrap/lib/stdlib/ebin/dict.beam +++ b/bootstrap/lib/stdlib/ebin/dict.beam diff --git a/bootstrap/lib/stdlib/ebin/digraph.beam b/bootstrap/lib/stdlib/ebin/digraph.beam Binary files differindex 60abe020fd..d584c51083 100644 --- a/bootstrap/lib/stdlib/ebin/digraph.beam +++ b/bootstrap/lib/stdlib/ebin/digraph.beam diff --git a/bootstrap/lib/stdlib/ebin/digraph_utils.beam b/bootstrap/lib/stdlib/ebin/digraph_utils.beam Binary files differindex fabe782d90..badb46d0af 100644 --- a/bootstrap/lib/stdlib/ebin/digraph_utils.beam +++ b/bootstrap/lib/stdlib/ebin/digraph_utils.beam diff --git a/bootstrap/lib/stdlib/ebin/edlin.beam b/bootstrap/lib/stdlib/ebin/edlin.beam Binary files differindex b6a7ecc9a5..473dcb6dc1 100644 --- a/bootstrap/lib/stdlib/ebin/edlin.beam +++ b/bootstrap/lib/stdlib/ebin/edlin.beam diff --git a/bootstrap/lib/stdlib/ebin/edlin_expand.beam b/bootstrap/lib/stdlib/ebin/edlin_expand.beam Binary files differindex 865066c3ac..f2572a212b 100644 --- a/bootstrap/lib/stdlib/ebin/edlin_expand.beam +++ b/bootstrap/lib/stdlib/ebin/edlin_expand.beam diff --git a/bootstrap/lib/stdlib/ebin/epp.beam b/bootstrap/lib/stdlib/ebin/epp.beam Binary files differindex e3d8c21edf..67c737dddd 100644 --- a/bootstrap/lib/stdlib/ebin/epp.beam +++ b/bootstrap/lib/stdlib/ebin/epp.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_abstract_code.beam b/bootstrap/lib/stdlib/ebin/erl_abstract_code.beam Binary files differindex 973ad9fb80..fefdd4f54a 100644 --- a/bootstrap/lib/stdlib/ebin/erl_abstract_code.beam +++ b/bootstrap/lib/stdlib/ebin/erl_abstract_code.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_anno.beam b/bootstrap/lib/stdlib/ebin/erl_anno.beam Binary files differindex a40fcb3f92..c4117509dd 100644 --- a/bootstrap/lib/stdlib/ebin/erl_anno.beam +++ b/bootstrap/lib/stdlib/ebin/erl_anno.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_bits.beam b/bootstrap/lib/stdlib/ebin/erl_bits.beam Binary files differindex 1beb0d5ebb..462b2677e1 100644 --- a/bootstrap/lib/stdlib/ebin/erl_bits.beam +++ b/bootstrap/lib/stdlib/ebin/erl_bits.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_compile.beam b/bootstrap/lib/stdlib/ebin/erl_compile.beam Binary files differindex 5dd5691b3b..ca55b187e8 100644 --- a/bootstrap/lib/stdlib/ebin/erl_compile.beam +++ b/bootstrap/lib/stdlib/ebin/erl_compile.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_error.beam b/bootstrap/lib/stdlib/ebin/erl_error.beam Binary files differindex 5d1b91aeef..0c3275465d 100644 --- a/bootstrap/lib/stdlib/ebin/erl_error.beam +++ b/bootstrap/lib/stdlib/ebin/erl_error.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_eval.beam b/bootstrap/lib/stdlib/ebin/erl_eval.beam Binary files differindex 5ef1521371..4743c2a730 100644 --- a/bootstrap/lib/stdlib/ebin/erl_eval.beam +++ b/bootstrap/lib/stdlib/ebin/erl_eval.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_expand_records.beam b/bootstrap/lib/stdlib/ebin/erl_expand_records.beam Binary files differindex e527f98776..7597146d22 100644 --- a/bootstrap/lib/stdlib/ebin/erl_expand_records.beam +++ b/bootstrap/lib/stdlib/ebin/erl_expand_records.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_internal.beam b/bootstrap/lib/stdlib/ebin/erl_internal.beam Binary files differindex 9015ad8936..820ff0e779 100644 --- a/bootstrap/lib/stdlib/ebin/erl_internal.beam +++ b/bootstrap/lib/stdlib/ebin/erl_internal.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_lint.beam b/bootstrap/lib/stdlib/ebin/erl_lint.beam Binary files differindex c73ecdf942..d56a730da1 100644 --- a/bootstrap/lib/stdlib/ebin/erl_lint.beam +++ b/bootstrap/lib/stdlib/ebin/erl_lint.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_parse.beam b/bootstrap/lib/stdlib/ebin/erl_parse.beam Binary files differindex 41369ca147..1c910f9f38 100644 --- a/bootstrap/lib/stdlib/ebin/erl_parse.beam +++ b/bootstrap/lib/stdlib/ebin/erl_parse.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_posix_msg.beam b/bootstrap/lib/stdlib/ebin/erl_posix_msg.beam Binary files differindex 5f6c03bcae..5696cbc072 100644 --- a/bootstrap/lib/stdlib/ebin/erl_posix_msg.beam +++ b/bootstrap/lib/stdlib/ebin/erl_posix_msg.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_pp.beam b/bootstrap/lib/stdlib/ebin/erl_pp.beam Binary files differindex 1965b9e656..1f6baa7653 100644 --- a/bootstrap/lib/stdlib/ebin/erl_pp.beam +++ b/bootstrap/lib/stdlib/ebin/erl_pp.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_scan.beam b/bootstrap/lib/stdlib/ebin/erl_scan.beam Binary files differindex 18538620b7..eac7242485 100644 --- a/bootstrap/lib/stdlib/ebin/erl_scan.beam +++ b/bootstrap/lib/stdlib/ebin/erl_scan.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_tar.beam b/bootstrap/lib/stdlib/ebin/erl_tar.beam Binary files differindex 871ccd82d4..dbffb05446 100644 --- a/bootstrap/lib/stdlib/ebin/erl_tar.beam +++ b/bootstrap/lib/stdlib/ebin/erl_tar.beam diff --git a/bootstrap/lib/stdlib/ebin/error_logger_file_h.beam b/bootstrap/lib/stdlib/ebin/error_logger_file_h.beam Binary files differindex 64de26f7a4..ce2d0a95d2 100644 --- a/bootstrap/lib/stdlib/ebin/error_logger_file_h.beam +++ b/bootstrap/lib/stdlib/ebin/error_logger_file_h.beam diff --git a/bootstrap/lib/stdlib/ebin/error_logger_tty_h.beam b/bootstrap/lib/stdlib/ebin/error_logger_tty_h.beam Binary files differindex e7ee41cc2e..66258d0fb9 100644 --- a/bootstrap/lib/stdlib/ebin/error_logger_tty_h.beam +++ b/bootstrap/lib/stdlib/ebin/error_logger_tty_h.beam diff --git a/bootstrap/lib/stdlib/ebin/escript.beam b/bootstrap/lib/stdlib/ebin/escript.beam Binary files differindex e846ca420e..69e3b7dddc 100644 --- a/bootstrap/lib/stdlib/ebin/escript.beam +++ b/bootstrap/lib/stdlib/ebin/escript.beam diff --git a/bootstrap/lib/stdlib/ebin/ets.beam b/bootstrap/lib/stdlib/ebin/ets.beam Binary files differindex 350316dfcc..fbd823cc39 100644 --- a/bootstrap/lib/stdlib/ebin/ets.beam +++ b/bootstrap/lib/stdlib/ebin/ets.beam diff --git a/bootstrap/lib/stdlib/ebin/eval_bits.beam b/bootstrap/lib/stdlib/ebin/eval_bits.beam Binary files differindex 08f86f86ff..063393d5d8 100644 --- a/bootstrap/lib/stdlib/ebin/eval_bits.beam +++ b/bootstrap/lib/stdlib/ebin/eval_bits.beam diff --git a/bootstrap/lib/stdlib/ebin/file_sorter.beam b/bootstrap/lib/stdlib/ebin/file_sorter.beam Binary files differindex 0a42a40d7e..a2e8fef834 100644 --- a/bootstrap/lib/stdlib/ebin/file_sorter.beam +++ b/bootstrap/lib/stdlib/ebin/file_sorter.beam diff --git a/bootstrap/lib/stdlib/ebin/filelib.beam b/bootstrap/lib/stdlib/ebin/filelib.beam Binary files differindex 62b8c3588e..fe62f6636c 100644 --- a/bootstrap/lib/stdlib/ebin/filelib.beam +++ b/bootstrap/lib/stdlib/ebin/filelib.beam diff --git a/bootstrap/lib/stdlib/ebin/filename.beam b/bootstrap/lib/stdlib/ebin/filename.beam Binary files differindex d353a11f3e..1461e1cbc8 100644 --- a/bootstrap/lib/stdlib/ebin/filename.beam +++ b/bootstrap/lib/stdlib/ebin/filename.beam diff --git a/bootstrap/lib/stdlib/ebin/gb_sets.beam b/bootstrap/lib/stdlib/ebin/gb_sets.beam Binary files differindex 8d1e2c11e0..ae19fe9d10 100644 --- a/bootstrap/lib/stdlib/ebin/gb_sets.beam +++ b/bootstrap/lib/stdlib/ebin/gb_sets.beam diff --git a/bootstrap/lib/stdlib/ebin/gb_trees.beam b/bootstrap/lib/stdlib/ebin/gb_trees.beam Binary files differindex 0aae509cb6..49fd69632f 100644 --- a/bootstrap/lib/stdlib/ebin/gb_trees.beam +++ b/bootstrap/lib/stdlib/ebin/gb_trees.beam diff --git a/bootstrap/lib/stdlib/ebin/gen.beam b/bootstrap/lib/stdlib/ebin/gen.beam Binary files differindex 85a9c25618..92e4e2dab8 100644 --- a/bootstrap/lib/stdlib/ebin/gen.beam +++ b/bootstrap/lib/stdlib/ebin/gen.beam diff --git a/bootstrap/lib/stdlib/ebin/gen_event.beam b/bootstrap/lib/stdlib/ebin/gen_event.beam Binary files differindex f3376d2ee2..45200c3ddd 100644 --- a/bootstrap/lib/stdlib/ebin/gen_event.beam +++ b/bootstrap/lib/stdlib/ebin/gen_event.beam diff --git a/bootstrap/lib/stdlib/ebin/gen_fsm.beam b/bootstrap/lib/stdlib/ebin/gen_fsm.beam Binary files differindex 5c4a0aa3d5..8f17cee54d 100644 --- a/bootstrap/lib/stdlib/ebin/gen_fsm.beam +++ b/bootstrap/lib/stdlib/ebin/gen_fsm.beam diff --git a/bootstrap/lib/stdlib/ebin/gen_server.beam b/bootstrap/lib/stdlib/ebin/gen_server.beam Binary files differindex 06c3d6de19..fea013f8cf 100644 --- a/bootstrap/lib/stdlib/ebin/gen_server.beam +++ b/bootstrap/lib/stdlib/ebin/gen_server.beam diff --git a/bootstrap/lib/stdlib/ebin/gen_statem.beam b/bootstrap/lib/stdlib/ebin/gen_statem.beam Binary files differindex 0aab776d07..0d7aa92875 100644 --- a/bootstrap/lib/stdlib/ebin/gen_statem.beam +++ b/bootstrap/lib/stdlib/ebin/gen_statem.beam diff --git a/bootstrap/lib/stdlib/ebin/io.beam b/bootstrap/lib/stdlib/ebin/io.beam Binary files differindex 631863b587..005c89a1e3 100644 --- a/bootstrap/lib/stdlib/ebin/io.beam +++ b/bootstrap/lib/stdlib/ebin/io.beam diff --git a/bootstrap/lib/stdlib/ebin/io_lib.beam b/bootstrap/lib/stdlib/ebin/io_lib.beam Binary files differindex 45692978e4..47201dba0c 100644 --- a/bootstrap/lib/stdlib/ebin/io_lib.beam +++ b/bootstrap/lib/stdlib/ebin/io_lib.beam diff --git a/bootstrap/lib/stdlib/ebin/io_lib_format.beam b/bootstrap/lib/stdlib/ebin/io_lib_format.beam Binary files differindex 66047b5070..c2f8a11e40 100644 --- a/bootstrap/lib/stdlib/ebin/io_lib_format.beam +++ b/bootstrap/lib/stdlib/ebin/io_lib_format.beam diff --git a/bootstrap/lib/stdlib/ebin/io_lib_fread.beam b/bootstrap/lib/stdlib/ebin/io_lib_fread.beam Binary files differindex 22690ce685..f81265ebd9 100644 --- a/bootstrap/lib/stdlib/ebin/io_lib_fread.beam +++ b/bootstrap/lib/stdlib/ebin/io_lib_fread.beam diff --git a/bootstrap/lib/stdlib/ebin/io_lib_pretty.beam b/bootstrap/lib/stdlib/ebin/io_lib_pretty.beam Binary files differindex b0f4177d71..d7bd6172e4 100644 --- a/bootstrap/lib/stdlib/ebin/io_lib_pretty.beam +++ b/bootstrap/lib/stdlib/ebin/io_lib_pretty.beam diff --git a/bootstrap/lib/stdlib/ebin/lists.beam b/bootstrap/lib/stdlib/ebin/lists.beam Binary files differindex 5a330bece0..9bacdd52b0 100644 --- a/bootstrap/lib/stdlib/ebin/lists.beam +++ b/bootstrap/lib/stdlib/ebin/lists.beam diff --git a/bootstrap/lib/stdlib/ebin/log_mf_h.beam b/bootstrap/lib/stdlib/ebin/log_mf_h.beam Binary files differindex b812621c0a..f4561164a3 100644 --- a/bootstrap/lib/stdlib/ebin/log_mf_h.beam +++ b/bootstrap/lib/stdlib/ebin/log_mf_h.beam diff --git a/bootstrap/lib/stdlib/ebin/maps.beam b/bootstrap/lib/stdlib/ebin/maps.beam Binary files differindex 193a06241d..a49a65714f 100644 --- a/bootstrap/lib/stdlib/ebin/maps.beam +++ b/bootstrap/lib/stdlib/ebin/maps.beam diff --git a/bootstrap/lib/stdlib/ebin/math.beam b/bootstrap/lib/stdlib/ebin/math.beam Binary files differindex 7e61673b35..e388b2c304 100644 --- a/bootstrap/lib/stdlib/ebin/math.beam +++ b/bootstrap/lib/stdlib/ebin/math.beam diff --git a/bootstrap/lib/stdlib/ebin/ms_transform.beam b/bootstrap/lib/stdlib/ebin/ms_transform.beam Binary files differindex bb98ae56b3..1a07331fe9 100644 --- a/bootstrap/lib/stdlib/ebin/ms_transform.beam +++ b/bootstrap/lib/stdlib/ebin/ms_transform.beam diff --git a/bootstrap/lib/stdlib/ebin/orddict.beam b/bootstrap/lib/stdlib/ebin/orddict.beam Binary files differindex d6f267ad8f..cc32d43f7a 100644 --- a/bootstrap/lib/stdlib/ebin/orddict.beam +++ b/bootstrap/lib/stdlib/ebin/orddict.beam diff --git a/bootstrap/lib/stdlib/ebin/ordsets.beam b/bootstrap/lib/stdlib/ebin/ordsets.beam Binary files differindex 44f9e5d8b5..4f2465c9bb 100644 --- a/bootstrap/lib/stdlib/ebin/ordsets.beam +++ b/bootstrap/lib/stdlib/ebin/ordsets.beam diff --git a/bootstrap/lib/stdlib/ebin/otp_internal.beam b/bootstrap/lib/stdlib/ebin/otp_internal.beam Binary files differindex 26d495af72..41e88c4eaa 100644 --- a/bootstrap/lib/stdlib/ebin/otp_internal.beam +++ b/bootstrap/lib/stdlib/ebin/otp_internal.beam diff --git a/bootstrap/lib/stdlib/ebin/pool.beam b/bootstrap/lib/stdlib/ebin/pool.beam Binary files differindex 408ee3a0c9..2b83887ba9 100644 --- a/bootstrap/lib/stdlib/ebin/pool.beam +++ b/bootstrap/lib/stdlib/ebin/pool.beam diff --git a/bootstrap/lib/stdlib/ebin/proc_lib.beam b/bootstrap/lib/stdlib/ebin/proc_lib.beam Binary files differindex b51898901d..c960faf2d3 100644 --- a/bootstrap/lib/stdlib/ebin/proc_lib.beam +++ b/bootstrap/lib/stdlib/ebin/proc_lib.beam diff --git a/bootstrap/lib/stdlib/ebin/proplists.beam b/bootstrap/lib/stdlib/ebin/proplists.beam Binary files differindex a8ddeb2d57..89371c49f4 100644 --- a/bootstrap/lib/stdlib/ebin/proplists.beam +++ b/bootstrap/lib/stdlib/ebin/proplists.beam diff --git a/bootstrap/lib/stdlib/ebin/qlc.beam b/bootstrap/lib/stdlib/ebin/qlc.beam Binary files differindex cca174e3cd..13ec8cfe1f 100644 --- a/bootstrap/lib/stdlib/ebin/qlc.beam +++ b/bootstrap/lib/stdlib/ebin/qlc.beam diff --git a/bootstrap/lib/stdlib/ebin/qlc_pt.beam b/bootstrap/lib/stdlib/ebin/qlc_pt.beam Binary files differindex 4d86fdfec2..cb4ea94478 100644 --- a/bootstrap/lib/stdlib/ebin/qlc_pt.beam +++ b/bootstrap/lib/stdlib/ebin/qlc_pt.beam diff --git a/bootstrap/lib/stdlib/ebin/queue.beam b/bootstrap/lib/stdlib/ebin/queue.beam Binary files differindex 94728e30cc..e6195d98c1 100644 --- a/bootstrap/lib/stdlib/ebin/queue.beam +++ b/bootstrap/lib/stdlib/ebin/queue.beam diff --git a/bootstrap/lib/stdlib/ebin/rand.beam b/bootstrap/lib/stdlib/ebin/rand.beam Binary files differindex 7f1e6140bf..0202b60efe 100644 --- a/bootstrap/lib/stdlib/ebin/rand.beam +++ b/bootstrap/lib/stdlib/ebin/rand.beam diff --git a/bootstrap/lib/stdlib/ebin/random.beam b/bootstrap/lib/stdlib/ebin/random.beam Binary files differindex a4bc2b6128..dee398f355 100644 --- a/bootstrap/lib/stdlib/ebin/random.beam +++ b/bootstrap/lib/stdlib/ebin/random.beam diff --git a/bootstrap/lib/stdlib/ebin/re.beam b/bootstrap/lib/stdlib/ebin/re.beam Binary files differindex e16327b906..3a1b44a49b 100644 --- a/bootstrap/lib/stdlib/ebin/re.beam +++ b/bootstrap/lib/stdlib/ebin/re.beam diff --git a/bootstrap/lib/stdlib/ebin/sets.beam b/bootstrap/lib/stdlib/ebin/sets.beam Binary files differindex e3609ea527..dd9009d114 100644 --- a/bootstrap/lib/stdlib/ebin/sets.beam +++ b/bootstrap/lib/stdlib/ebin/sets.beam diff --git a/bootstrap/lib/stdlib/ebin/shell.beam b/bootstrap/lib/stdlib/ebin/shell.beam Binary files differindex 319bb9aebf..66409894dd 100644 --- a/bootstrap/lib/stdlib/ebin/shell.beam +++ b/bootstrap/lib/stdlib/ebin/shell.beam diff --git a/bootstrap/lib/stdlib/ebin/shell_default.beam b/bootstrap/lib/stdlib/ebin/shell_default.beam Binary files differindex afd0c65921..ff4d1071d5 100644 --- a/bootstrap/lib/stdlib/ebin/shell_default.beam +++ b/bootstrap/lib/stdlib/ebin/shell_default.beam diff --git a/bootstrap/lib/stdlib/ebin/slave.beam b/bootstrap/lib/stdlib/ebin/slave.beam Binary files differindex 259d300155..6cbc413f9d 100644 --- a/bootstrap/lib/stdlib/ebin/slave.beam +++ b/bootstrap/lib/stdlib/ebin/slave.beam diff --git a/bootstrap/lib/stdlib/ebin/sofs.beam b/bootstrap/lib/stdlib/ebin/sofs.beam Binary files differindex 3a90a07add..ee8dddf2cf 100644 --- a/bootstrap/lib/stdlib/ebin/sofs.beam +++ b/bootstrap/lib/stdlib/ebin/sofs.beam diff --git a/bootstrap/lib/stdlib/ebin/string.beam b/bootstrap/lib/stdlib/ebin/string.beam Binary files differindex ee19fb41a4..0a48eb1a46 100644 --- a/bootstrap/lib/stdlib/ebin/string.beam +++ b/bootstrap/lib/stdlib/ebin/string.beam diff --git a/bootstrap/lib/stdlib/ebin/supervisor.beam b/bootstrap/lib/stdlib/ebin/supervisor.beam Binary files differindex 1816fe6c41..9eecd35256 100644 --- a/bootstrap/lib/stdlib/ebin/supervisor.beam +++ b/bootstrap/lib/stdlib/ebin/supervisor.beam diff --git a/bootstrap/lib/stdlib/ebin/supervisor_bridge.beam b/bootstrap/lib/stdlib/ebin/supervisor_bridge.beam Binary files differindex 761907b40c..f05aaef893 100644 --- a/bootstrap/lib/stdlib/ebin/supervisor_bridge.beam +++ b/bootstrap/lib/stdlib/ebin/supervisor_bridge.beam diff --git a/bootstrap/lib/stdlib/ebin/sys.beam b/bootstrap/lib/stdlib/ebin/sys.beam Binary files differindex a3f46b5983..4158d032c2 100644 --- a/bootstrap/lib/stdlib/ebin/sys.beam +++ b/bootstrap/lib/stdlib/ebin/sys.beam diff --git a/bootstrap/lib/stdlib/ebin/timer.beam b/bootstrap/lib/stdlib/ebin/timer.beam Binary files differindex 2953299e8b..f018f4f750 100644 --- a/bootstrap/lib/stdlib/ebin/timer.beam +++ b/bootstrap/lib/stdlib/ebin/timer.beam diff --git a/bootstrap/lib/stdlib/ebin/unicode.beam b/bootstrap/lib/stdlib/ebin/unicode.beam Binary files differindex d61ed589aa..5f483b8358 100644 --- a/bootstrap/lib/stdlib/ebin/unicode.beam +++ b/bootstrap/lib/stdlib/ebin/unicode.beam diff --git a/bootstrap/lib/stdlib/ebin/unicode_util.beam b/bootstrap/lib/stdlib/ebin/unicode_util.beam Binary files differindex c0523f2e52..182472b5d9 100644 --- a/bootstrap/lib/stdlib/ebin/unicode_util.beam +++ b/bootstrap/lib/stdlib/ebin/unicode_util.beam diff --git a/bootstrap/lib/stdlib/ebin/uri_string.beam b/bootstrap/lib/stdlib/ebin/uri_string.beam Binary files differindex 7255fdca63..51c823eb51 100644 --- a/bootstrap/lib/stdlib/ebin/uri_string.beam +++ b/bootstrap/lib/stdlib/ebin/uri_string.beam diff --git a/bootstrap/lib/stdlib/ebin/win32reg.beam b/bootstrap/lib/stdlib/ebin/win32reg.beam Binary files differindex 136b65f9fb..dc6246f9ca 100644 --- a/bootstrap/lib/stdlib/ebin/win32reg.beam +++ b/bootstrap/lib/stdlib/ebin/win32reg.beam diff --git a/bootstrap/lib/stdlib/ebin/zip.beam b/bootstrap/lib/stdlib/ebin/zip.beam Binary files differindex 75c2289f47..751707a27c 100644 --- a/bootstrap/lib/stdlib/ebin/zip.beam +++ b/bootstrap/lib/stdlib/ebin/zip.beam diff --git a/erts/Makefile b/erts/Makefile index e62c896170..12ac50f7d5 100644 --- a/erts/Makefile +++ b/erts/Makefile @@ -153,3 +153,5 @@ release_docs: .PHONY: xmllint xmllint: $(MAKE) -C doc/src $@ + +include $(ERL_TOP)/make/app_targets.mk diff --git a/erts/aclocal.m4 b/erts/aclocal.m4 index 0ca2755802..e2b7e1eada 100644 --- a/erts/aclocal.m4 +++ b/erts/aclocal.m4 @@ -2988,7 +2988,7 @@ case $host_os in DED_LDFLAGS="-64 $DED_LDFLAGS" fi ;; - aix4*) + aix*|os400*) DED_LDFLAGS="-G -bnoentry -bexpall" ;; freebsd2*) diff --git a/erts/configure.in b/erts/configure.in index 716dc7bd27..122a90557b 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -447,6 +447,12 @@ dnl --------------------------------------------------------------------- dnl NOTE: CPPFLAGS will be included in CFLAGS at the end case $host_os in linux*) CPPFLAGS="$CPPFLAGS -D_GNU_SOURCE";; + aix*|os400*) + # * _ALL_SOURCE: Required to get the winsize structure for TIOCSWINSZ. + # * _LINUX_SOURCE_COMPAT: Not required, but makes some libc functions + # behave closer to glibc assumptions. + CPPFLAGS="$CPPFLAGS -D_ALL_SOURCE -D_LINUX_SOURCE_COMPAT" + ;; win32) # The ethread library requires _WIN32_WINNT of at least 0x0403. # -D_WIN32_WINNT=* from CPPFLAGS is saved in ETHR_DEFS. @@ -966,7 +972,7 @@ AC_SUBST(ERLANG_OSTYPE) AC_MSG_CHECKING(for extra flags needed to export symbols) DEXPORT="" case $host_os in - aix4*) + aix*|os400*) DEXPORT=-Wl,-bexpall,-brtl ;; bsdi*) @@ -1427,7 +1433,7 @@ if test "$have_gethostbyname_r" = yes; then AC_DEFINE(HAVE_GETHOSTBYNAME_R, GHBN_R_SOLARIS, [Define to flavour of gethostbyname_r]) ;; - aix4*) + aix*|os400*) # AIX version also needs "struct hostent_data" defn AC_TRY_COMPILE([#include <netdb.h>], [struct hostent_data hd;], @@ -2036,7 +2042,7 @@ AC_CHECK_FUNCS([ieee_handler fpsetmask finite isnan isinf res_gethostbyname dlop gethrtime localtime_r gmtime_r inet_pton mprotect \ mmap mremap memcpy mallopt sbrk _sbrk __sbrk brk _brk __brk \ flockfile fstat strlcpy strlcat setsid posix2time time2posix \ - setlocale nl_langinfo poll mlockall ppoll]) + setlocale nl_langinfo poll mlockall ppoll vsyslog]) AC_MSG_CHECKING([for isfinite]) AC_TRY_LINK([#include <math.h>], diff --git a/erts/doc/src/erl_dist_protocol.xml b/erts/doc/src/erl_dist_protocol.xml index f924c8a70b..0c1100d394 100644 --- a/erts/doc/src/erl_dist_protocol.xml +++ b/erts/doc/src/erl_dist_protocol.xml @@ -109,7 +109,8 @@ <title>Register a Node in EPMD</title> <p>When a distributed node is started it registers itself in the EPMD. The message <c>ALIVE2_REQ</c> described below is sent from the node to - the EPMD. The response from the EPMD is <c>ALIVE2_RESP</c>.</p> + the EPMD. The response from the EPMD is <c>ALIVE2_X_RESP</c> (or + <c>ALIVE2_RESP</c>).</p> <table align="left"> <row> @@ -155,12 +156,12 @@ <tag><c>HighestVersion</c></tag> <item> <p>The highest distribution version that this node can handle. - The value in Erlang/OTP R6B and later is 5.</p> + The value in OTP 23 and later is 6.</p> </item> <tag><c>LowestVersion</c></tag> <item> <p>The lowest distribution version that this node can handle. - The value in Erlang/OTP R6B and later is 5.</p> + The value in OTP 23 and later is 5.</p> </item> <tag><c>Nlen</c></tag> <item> @@ -184,7 +185,24 @@ node is a distributed node. When the connection is closed, the node is automatically unregistered from the EPMD.</p> - <p>The response message <c>ALIVE2_RESP</c> is as follows:</p> + <p>The response message is either <c>ALIVE2_X_RESP</c> or + <c>ALIVE2_RESP</c> depending on distribution version. If both the node + and EPMD support distribution version 6 then response is + <c>ALIVE2_X_RESP</c> otherwise it is the older <c>ALIVE2_RESP</c>:</p> + + <table align="left"> + <row> + <cell align="center">1</cell> + <cell align="center">1</cell> + <cell align="center">4</cell> + </row> + <row> + <cell align="center"><c>118</c></cell> + <cell align="center"><c>Result</c></cell> + <cell align="center"><c>Creation</c></cell> + </row> + <tcaption>ALIVE2_X_RESP (118) with 32 bit creation</tcaption> + </table> <table align="left"> <row> @@ -197,7 +215,7 @@ <cell align="center"><c>Result</c></cell> <cell align="center"><c>Creation</c></cell> </row> - <tcaption>ALIVE2_RESP (121)</tcaption> + <tcaption>ALIVE2_RESP (121) with 16-bit creation</tcaption> </table> <p>Result = 0 -> ok, result > 0 -> error.</p> @@ -793,7 +811,8 @@ DiB == gen_digest(ChA, ICA)? </item> <tag><c>-define(DFLAG_NEW_FUN_TAGS,16#80).</c></tag> <item> - <p>The node understand new fun tags.</p> + <p>The node understands the <seealso marker="erl_ext_dist#NEW_FUN_EXT"> + <c>NEW_FUN_EXT</c></seealso> tag.</p> </item> <tag><c>-define(DFLAG_EXTENDED_PIDS_PORTS,16#100).</c></tag> <item> @@ -802,13 +821,18 @@ DiB == gen_digest(ChA, ICA)? </item> <tag><c>-define(DFLAG_EXPORT_PTR_TAG,16#200).</c></tag> <item> + <p>The node understands the <seealso marker="erl_ext_dist#EXPORT_EXT"> + <c>EXPORT_EXT</c></seealso> tag.</p> </item> <tag><c>-define(DFLAG_BIT_BINARIES,16#400).</c></tag> <item> + <p>The node understands the <seealso marker="erl_ext_dist#BIT_BINARY_EXT"> + <c>BIT_BINARY_EXT</c></seealso> tag.</p> </item> <tag><c>-define(DFLAG_NEW_FLOATS,16#800).</c></tag> <item> - <p>The node understands new float format.</p> + <p>The node understands the <seealso marker="erl_ext_dist#NEW_FLOAT_EXT"> + <c>NEW_FLOAT_EXT</c></seealso> tag.</p> </item> <tag><c>-define(DFLAG_UNICODE_IO,16#1000).</c></tag> <item> @@ -817,21 +841,34 @@ DiB == gen_digest(ChA, ICA)? <item> <p>The node implements atom cache in distribution header.</p> </item> + <marker id="DFLAG_SMALL_ATOM_TAGS"/> <tag><c>-define(DFLAG_SMALL_ATOM_TAGS, 16#4000).</c></tag> <item> - <p>The node understand the <c>SMALL_ATOM_EXT</c> tag.</p> + <p>The node understands the <seealso marker="erl_ext_dist#SMALL_ATOM_EXT"> + <c>SMALL_ATOM_EXT</c></seealso> tag.</p> </item> + <marker id="DFLAG_UTF8_ATOMS"/> <tag><c>-define(DFLAG_UTF8_ATOMS, 16#10000).</c></tag> <item> - <p>The node understand UTF-8 encoded atoms.</p> + <p>The node understands UTF-8 atoms encoded with + <seealso marker="erl_ext_dist#ATOM_UTF8_EXT"> + <c>ATOM_UTF8_EXT</c></seealso> and + <seealso marker="erl_ext_dist#SMALL_ATOM_UTF8_EXT"> + <c>SMALL ATOM_UTF8_EXT</c></seealso>.</p> </item> <tag><c>-define(DFLAG_MAP_TAG, 16#20000).</c></tag> <item> - <p>The node understand the map tag.</p> + <p>The node understands the map tag + <seealso marker="erl_ext_dist#MAP_EXT"><c>MAP_EXT</c></seealso>.</p> </item> + <marker id="DFLAG_BIG_CREATION"/> <tag><c>-define(DFLAG_BIG_CREATION, 16#40000).</c></tag> <item> - <p>The node understand big node creation.</p> + <p>The node understands big node creation tags + <seealso marker="erl_ext_dist#NEW_PID_EXT"><c>NEW_PID_EXT</c></seealso>, + <seealso marker="erl_ext_dist#NEW_PORT_EXT"><c>NEW_PORT_EXT</c></seealso> and + <seealso marker="erl_ext_dist#NEWER_REFERENCE_EXT"><c>NEWER_REFERENCE_EXT</c></seealso>. + </p> </item> <tag><c>-define(DFLAG_SEND_SENDER, 16#80000).</c></tag> <item> @@ -855,6 +892,7 @@ DiB == gen_digest(ChA, ICA)? <seealso marker="#control_message">control message</seealso>s instead of the non-PAYLOAD variants.</p> </item> + <marker id="DFLAG_FRAGMENTS"/> <tag><c>-define(DFLAG_FRAGMENTS, 16#800000).</c></tag> <item> <p>Use <seealso marker="erl_ext_dist#fragments">fragmented</seealso> diff --git a/erts/doc/src/erl_ext_dist.xml b/erts/doc/src/erl_ext_dist.xml index 2ba5994557..c5b2ce1a0a 100644 --- a/erts/doc/src/erl_ext_dist.xml +++ b/erts/doc/src/erl_ext_dist.xml @@ -264,7 +264,7 @@ consists of. Length is a 2 byte big-endian integer if flag <c>LongAtoms</c> has been set, otherwise a 1 byte integer. When distribution flag - <seealso marker="erl_dist_protocol#dflags"> + <seealso marker="erl_dist_protocol#DFLAG_UTF8_ATOMS"> <c>DFLAG_UTF8_ATOMS</c></seealso> has been exchanged between both nodes in the <seealso marker="erl_dist_protocol#distribution_handshake"> @@ -316,8 +316,8 @@ </p> <p>Fragmented distribution messages are only used if the receiving node signals that it supports them via the - <seealso marker="erl_dist_protocol#dflags">DFLAG_FRAGMENTS</seealso> distribution - flag.</p> + <seealso marker="erl_dist_protocol#DFLAG_FRAGMENTS">DFLAG_FRAGMENTS</seealso> + distribution flag.</p> <p>A process must complete the sending of a fragmented message before it can start sending any other message on the same distribution channel.</p> @@ -637,11 +637,14 @@ <seealso marker="#NEW_PID_EXT"><c>NEW_PID_EXT</c></seealso>. Port operations are not allowed across node boundaries. </p> - <p>Introduced in OTP 19, but only to be decoded and echoed back. Not - encoded for local ports. Planned to supersede <seealso marker="#PORT_EXT"> - <c>PORT_EXT</c></seealso> in OTP 23 when - <seealso marker="erl_dist_protocol#dflags"><c>DFLAG_BIG_CREATON</c></seealso> - becomes mandatory. + <p><c>NEW_PORT_EXT</c> was introduced in OTP 19, but only to be decoded + and echoed back. Not encoded for local ports. + </p> + <p>In OTP 23 distribution flag + <seealso marker="erl_dist_protocol#DFLAG_BIG_CREATION"><c>DFLAG_BIG_CREATION</c></seealso> + became mandatory. All ports are now + encoded using <c>NEW_PORT_EXT</c>, even external ports received as <seealso + marker="#PORT_EXT"><c>PORT_EXT</c></seealso> from older nodes. </p> </section> @@ -719,11 +722,14 @@ erlang:list_to_pid/1</seealso>).</p> </item> </taglist> - <p>Introduced in OTP 19, but only to be decoded and echoed back. Not - encoded for local processes. Planned to supersede <seealso marker="#PID_EXT"> - <c>PID_EXT</c></seealso> in OTP 23 when - <seealso marker="erl_dist_protocol#dflags"><c>DFLAG_BIG_CREATON</c></seealso> - becomes mandatory. + <p><c>NEW_PID_EXT</c> was introduced in OTP 19, but only to be decoded + and echoed back. Not encoded for local processes. + </p> + <p>In OTP 23 distribution flag + <seealso marker="erl_dist_protocol#DFLAG_BIG_CREATION"><c>DFLAG_BIG_CREATION</c></seealso> + became mandatory. All pids are now encoded using <c>NEW_PID_EXT</c>, + even external pids received as + <seealso marker="#PID_EXT"><c>PID_EXT</c></seealso> from older nodes. </p> </section> @@ -1047,11 +1053,15 @@ <seealso marker="#NEW_PID_EXT"><c>NEW_PID_EXT</c></seealso>.</p> </item> </taglist> - <p>Introduced in OTP 19, but only to be decoded and echoed back. Not - encoded for local references. Planned to supersede <seealso marker="#NEW_REFERENCE_EXT"> - <c>NEW_REFERENCE_EXT</c></seealso> in OTP 23 when - <seealso marker="erl_dist_protocol#dflags"><c>DFLAG_BIG_CREATON</c></seealso> - becomes mandatory. + <p><c>NEWER_REFERENCE_EXT</c> was introduced in OTP 19, but only to be decoded + and echoed back. Not encoded for local references. + </p> + <p>In OTP 23 distribution flag + <seealso marker="erl_dist_protocol#DFLAG_BIG_CREATION"><c>DFLAG_BIG_CREATION</c></seealso> + became mandatory. All references are now encoded using + <c>NEWER_REFERENCE_EXT</c>, even external references received as + <seealso marker="#NEW_REFERENCE_EXT"><c>NEW_REFERENCE_EXT</c></seealso> + from older nodes. </p> </section> @@ -1408,7 +1418,7 @@ <p> <c>SMALL_ATOM_EXT</c> was introduced in ERTS 5.7.2 and require an exchange of distribution flag - <seealso marker="erl_dist_protocol#dflags"> + <seealso marker="erl_dist_protocol#DFLAG_SMALL_ATOM_TAGS"> <c>DFLAG_SMALL_ATOM_TAGS</c></seealso> in the <seealso marker="erl_dist_protocol#distribution_handshake"> distribution handshake</seealso>. diff --git a/erts/doc/src/erl_ext_fig.gif b/erts/doc/src/erl_ext_fig.gif Binary files differindex 14d6bbc871..40dd17bd5e 100644 --- a/erts/doc/src/erl_ext_fig.gif +++ b/erts/doc/src/erl_ext_fig.gif diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index 2183f75487..376deb2253 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -337,6 +337,16 @@ Z = erlang:adler32_combine(X,Y,iolist_size(Data2)).</code> </func> <func> + <name name="atom_to_binary" arity="1" since="OTP @OTP-15995@"/> + <fsummary>Return the binary representation of an atom.</fsummary> + <desc> + <p> + The same as <seealso marker="#atom_to_binary/2"><c>atom_to_binary</c> + </seealso><c>(<anno>Atom</anno>, utf8)</c>.</p> + </desc> + </func> + + <func> <name name="atom_to_binary" arity="2" since=""/> <fsummary>Return the binary representation of an atom.</fsummary> <desc> @@ -409,14 +419,21 @@ Z = erlang:adler32_combine(X,Y,iolist_size(Data2)).</code> </func> <func> + <name name="binary_to_atom" arity="1" since="OTP @OTP-15995@"/> + <fsummary>Convert from text representation to an atom.</fsummary> + <desc> + <p> + The same as <seealso marker="#binary_to_atom/2"><c>binary_to_atom</c> + </seealso><c>(<anno>Binary</anno>, utf8)</c>.</p> + </desc> + </func> + + <func> <name name="binary_to_atom" arity="2" since=""/> <fsummary>Convert from text representation to an atom.</fsummary> <desc> <p>Returns the atom whose text representation is - <c><anno>Binary</anno></c>. - If <c><anno>Encoding</anno></c> is <c>latin1</c>, no - translation of bytes in the binary is done. - If <c><anno>Encoding</anno></c> + <c><anno>Binary</anno></c>. If <c><anno>Encoding</anno></c> is <c>utf8</c> or <c>unicode</c>, the binary must contain valid UTF-8 sequences.</p> <note> @@ -438,6 +455,17 @@ Z = erlang:adler32_combine(X,Y,iolist_size(Data2)).</code> </func> <func> + <name name="binary_to_existing_atom" arity="1" since="OTP @OTP-15995@"/> + <fsummary>Convert from text representation to an atom.</fsummary> + <desc> + <p> + The same as <seealso marker="#binary_to_existing_atom/2"> + <c>binary_to_existing_atom</c></seealso> + <c>(<anno>Binary</anno>, utf8)</c>.</p> + </desc> + </func> + + <func> <name name="binary_to_existing_atom" arity="2" since=""/> <fsummary>Convert from text representation to an atom.</fsummary> <desc> diff --git a/erts/doc/src/socket.xml b/erts/doc/src/socket.xml index 6b167e7203..708f87edf6 100644 --- a/erts/doc/src/socket.xml +++ b/erts/doc/src/socket.xml @@ -953,4 +953,3 @@ server(Addr, Port) -> </code> </section> </erlref> - diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 7f50372532..70718575e9 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -599,7 +599,6 @@ endif $(TTF_DIR)/erl_bif_table.c \ $(TTF_DIR)/erl_bif_table.h \ -$(TTF_DIR)/erl_bif_wrap.c \ $(TTF_DIR)/erl_bif_list.h \ $(TTF_DIR)/erl_atom_table.c \ $(TTF_DIR)/erl_atom_table.h \ @@ -885,7 +884,6 @@ RUN_OBJS += \ $(OBJDIR)/erl_bif_persistent.o \ $(OBJDIR)/erl_bif_atomics.o $(OBJDIR)/erl_bif_counters.o \ $(OBJDIR)/erl_bif_trace.o $(OBJDIR)/erl_bif_unique.o \ - $(OBJDIR)/erl_bif_wrap.o $(OBJDIR)/erl_nfunc_sched.o \ $(OBJDIR)/erl_guard_bifs.o $(OBJDIR)/erl_dirty_bif_wrap.o \ $(OBJDIR)/erl_trace.o $(OBJDIR)/copy.o \ $(OBJDIR)/utils.o $(OBJDIR)/bif.o \ @@ -920,7 +918,8 @@ RUN_OBJS += \ $(OBJDIR)/erl_ptab.o $(OBJDIR)/erl_map.o \ $(OBJDIR)/erl_msacc.o $(OBJDIR)/erl_lock_flags.o \ $(OBJDIR)/erl_io_queue.o $(OBJDIR)/erl_db_catree.o \ - $(ESOCK_RUN_OBJS) $(OBJDIR)/erl_flxctr.o + $(ESOCK_RUN_OBJS) $(OBJDIR)/erl_flxctr.o \ + $(OBJDIR)/erl_nfunc_sched.o LTTNG_OBJS = $(OBJDIR)/erlang_lttng.o @@ -1290,3 +1289,5 @@ ifndef VOID_EMULATOR endif endif endif + +include $(ERL_TOP)/make/app_targets.mk diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 93ba56dccd..4c67c2029f 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -404,6 +404,7 @@ atom min_bin_vheap_size atom minor atom minor_version atom Minus='-' +atom MinusMinus='--' atom module atom module_info atom monitored_by @@ -494,6 +495,7 @@ atom packet atom packet_size atom parallelism atom Plus='+' +atom PlusPlus='++' atom pause atom pending atom pending_driver diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 4c8ee5178a..dcbff99f54 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -835,21 +835,25 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) */ num_exps = export_list_size(code_ix); for (i = 0; i < num_exps; i++) { - Export *ep = export_list(i,code_ix); - if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) { - continue; - } - if (ep->beam[1] != 0) { - ep->addressv[code_ix] = (void *) ep->beam[1]; - ep->beam[1] = 0; - } else { - if (ep->addressv[code_ix] == ep->beam && - BeamIsOpCode(ep->beam[0], op_apply_bif)) { - continue; - } - ep->addressv[code_ix] = ep->beam; - ep->beam[0] = BeamOpCodeAddr(op_call_error_handler); - } + Export *ep = export_list(i, code_ix); + + if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) { + continue; + } + + DBG_CHECK_EXPORT(ep, code_ix); + + if (ep->trampoline.not_loaded.deferred != 0) { + ep->addressv[code_ix] = (void*)ep->trampoline.not_loaded.deferred; + ep->trampoline.not_loaded.deferred = 0; + } else { + if (ep->bif_table_index != -1) { + continue; + } + + ep->addressv[code_ix] = ep->trampoline.raw; + ep->trampoline.op = BeamOpCodeAddr(op_call_error_handler); + } } modp->curr.code_hdr->on_load_function_ptr = NULL; @@ -872,10 +876,11 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) { continue; } - if (BeamIsOpCode(ep->beam[0], op_apply_bif)) { + if (ep->bif_table_index != -1) { continue; } - ep->beam[1] = 0; + + ep->trampoline.not_loaded.deferred = 0; } } erts_release_code_write_permission(); @@ -1125,16 +1130,15 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls) mod_size = modp->old.code_length; /* - * Check if current instruction or continuation pointer points into module. + * Check if the instruction pointer points into module. */ - if (ErtsInArea(rp->i, mod_start, mod_size) - || ErtsInArea(rp->cp, mod_start, mod_size)) { + if (ErtsInArea(rp->i, mod_start, mod_size)) { return am_true; } - + *redsp += 1; - if (erts_check_nif_export_in_area(rp, mod_start, mod_size)) + if (erts_check_nfunc_in_area(rp, mod_start, mod_size)) return am_true; *redsp += (STACK_START(rp) - rp->stop) / 32; @@ -1885,25 +1889,28 @@ delete_code(Module* modp) for (i = 0; i < num_exps; i++) { Export *ep = export_list(i, code_ix); if (ep != NULL && (ep->info.mfa.module == module)) { - if (ep->addressv[code_ix] == ep->beam) { - if (BeamIsOpCode(ep->beam[0], op_apply_bif)) { - continue; - } - else if (BeamIsOpCode(ep->beam[0], op_i_generic_breakpoint)) { + if (ep->addressv[code_ix] == ep->trampoline.raw) { + if (BeamIsOpCode(ep->trampoline.op, op_i_generic_breakpoint)) { ERTS_LC_ASSERT(erts_thr_progress_is_blocking()); ASSERT(modp->curr.num_traced_exports > 0); DBG_TRACE_MFA_P(&ep->info.mfa, "export trace cleared, code_ix=%d", code_ix); - erts_clear_export_break(modp, &ep->info); + erts_clear_export_break(modp, ep); } else { - ASSERT(BeamIsOpCode(ep->beam[0], op_call_error_handler) || + ASSERT(BeamIsOpCode(ep->trampoline.op, op_call_error_handler) || !erts_initialized); } } - ep->addressv[code_ix] = ep->beam; - ep->beam[0] = BeamOpCodeAddr(op_call_error_handler); - ep->beam[1] = 0; + + if (ep->bif_table_index != -1 && ep->is_bif_traced) { + /* Code unloading kills both global and local call tracing. */ + ep->is_bif_traced = 0; + } + + ep->addressv[code_ix] = ep->trampoline.raw; + ep->trampoline.op = BeamOpCodeAddr(op_call_error_handler); + ep->trampoline.not_loaded.deferred = 0; DBG_TRACE_MFA_P(&ep->info.mfa, "export invalidation, code_ix=%d", code_ix); } diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 0832b3f374..1bb20f6ae3 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -207,9 +207,6 @@ erts_bp_match_functions(BpFunctions* f, ErtsCodeMFA *mfa, int specified) if (erts_is_function_native(ci)) { continue; } - if (is_nil(ci->mfa.module)) { /* Ignore BIF stub */ - continue; - } switch (specified) { case 3: if (ci->mfa.arity != mfa->arity) @@ -244,8 +241,10 @@ erts_bp_match_export(BpFunctions* f, ErtsCodeMFA *mfa, int specified) f->matching = (BpFunction *) Alloc(num_exps*sizeof(BpFunction)); ne = 0; for (i = 0; i < num_exps; i++) { - Export* ep = export_list(i, code_ix); - BeamInstr* pc; + BeamInstr *func; + Export* ep; + + ep = export_list(i, code_ix); switch (specified) { case 3: @@ -263,19 +262,20 @@ erts_bp_match_export(BpFunctions* f, ErtsCodeMFA *mfa, int specified) ASSERT(0); } - pc = ep->beam; - if (ep->addressv[code_ix] == pc) { - if (BeamIsOpCode(*pc, op_apply_bif) || - BeamIsOpCode(*pc, op_call_error_handler)) { - continue; - } - ASSERT(BeamIsOpCode(*pc, op_i_generic_breakpoint)); - } else if (erts_is_function_native(erts_code_to_codeinfo(ep->addressv[code_ix]))) { - continue; - } + func = ep->addressv[code_ix]; + + if (func == ep->trampoline.raw) { + if (BeamIsOpCode(*func, op_call_error_handler)) { + continue; + } + ASSERT(BeamIsOpCode(*func, op_i_generic_breakpoint)); + } else if (erts_is_function_native(erts_code_to_codeinfo(func))) { + continue; + } f->matching[ne].ci = &ep->info; f->matching[ne].mod = erts_get_module(ep->info.mfa.module, code_ix); + ne++; } @@ -305,18 +305,6 @@ erts_consolidate_bp_data(BpFunctions* f, int local) } } -void -erts_consolidate_bif_bp_data(void) -{ - int i; - - ERTS_LC_ASSERT(erts_has_code_write_permission()); - for (i = 0; i < BIF_SIZE; i++) { - Export *ep = bif_export[i]; - consolidate_bp_data(0, &ep->info, 0); - } -} - static void consolidate_bp_data(Module* modp, ErtsCodeInfo *ci, int local) { @@ -495,7 +483,7 @@ erts_set_mtrace_break(BpFunctions* f, Binary *match_spec, ErtsTracer tracer) } void -erts_set_call_trace_bif(ErtsCodeInfo *ci, Binary *match_spec, int local) +erts_set_export_trace(ErtsCodeInfo *ci, Binary *match_spec, int local) { Uint flags = local ? ERTS_BPF_LOCAL_TRACE : ERTS_BPF_GLOBAL_TRACE; @@ -503,25 +491,6 @@ erts_set_call_trace_bif(ErtsCodeInfo *ci, Binary *match_spec, int local) } void -erts_set_mtrace_bif(ErtsCodeInfo *ci, Binary *match_spec, ErtsTracer tracer) -{ - set_function_break(ci, match_spec, ERTS_BPF_META_TRACE, 0, tracer); -} - -void -erts_set_time_trace_bif(ErtsCodeInfo *ci, enum erts_break_op count_op) -{ - set_function_break(ci, NULL, - ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE, - count_op, erts_tracer_nil); -} - -void -erts_clear_time_trace_bif(ErtsCodeInfo *ci) { - clear_function_break(ci, ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE); -} - -void erts_set_debug_break(BpFunctions* f) { set_break(f, NULL, ERTS_BPF_DEBUG, 0, erts_tracer_nil); } @@ -547,7 +516,7 @@ erts_clear_trace_break(BpFunctions* f) } void -erts_clear_call_trace_bif(ErtsCodeInfo *ci, int local) +erts_clear_export_trace(ErtsCodeInfo *ci, int local) { GenericBp* g = ci->u.gen_bp; @@ -566,12 +535,6 @@ erts_clear_mtrace_break(BpFunctions* f) } void -erts_clear_mtrace_bif(ErtsCodeInfo *ci) -{ - clear_function_break(ci, ERTS_BPF_META_TRACE); -} - -void erts_clear_debug_break(BpFunctions* f) { ERTS_LC_ASSERT(erts_thr_progress_is_blocking()); @@ -630,58 +593,56 @@ erts_clear_module_break(Module *modp) { } void -erts_clear_export_break(Module* modp, ErtsCodeInfo *ci) +erts_clear_export_break(Module* modp, Export *ep) { + ErtsCodeInfo *ci; + ERTS_LC_ASSERT(erts_thr_progress_is_blocking()); + ci = &ep->info; + + ASSERT(erts_codeinfo_to_code(ci) == ep->trampoline.raw); + + ASSERT(BeamIsOpCode(ep->trampoline.op, op_i_generic_breakpoint)); + ep->trampoline.op = 0; + clear_function_break(ci, ERTS_BPF_ALL); erts_commit_staged_bp(); - *erts_codeinfo_to_code(ci) = (BeamInstr) 0; + consolidate_bp_data(modp, ci, 0); ASSERT(ci->u.gen_bp == NULL); } /* - * If c_p->cp is a trace return instruction, we set cp - * to be the place where we again start to execute code. + * If the topmost continuation pointer on the stack is a trace return + * instruction, we modify it to be the place where we again start to + * execute code. * - * cp is used by match spec {caller} to get the calling - * function, and if we don't do this fixup it will be - * 'undefined'. This has the odd side effect of {caller} - * not really being which function is the caller, but - * rather which function we are about to return to. + * This continuation pointer is used by match spec {caller} to get the + * calling function, and if we don't do this fixup it will be + * 'undefined'. This has the odd side effect of {caller} not really + * being the function which is the caller, but rather the function + * which we are about to return to. */ static void fixup_cp_before_trace(Process *c_p, int *return_to_trace) { - Eterm *cpp, *E = c_p->stop; - BeamInstr w = *c_p->cp; - if (BeamIsOpCode(w, op_return_trace)) { - cpp = &E[2]; - } else if (BeamIsOpCode(w, op_i_return_to_trace)) { - *return_to_trace = 1; - cpp = &E[0]; - } else if (BeamIsOpCode(w, op_i_return_time_trace)) { - cpp = &E[0]; - } else { - cpp = NULL; - } - if (cpp) { - for (;;) { - BeamInstr w = *cp_val(*cpp); - if (BeamIsOpCode(w, op_return_trace)) { - cpp += 3; - } else if (BeamIsOpCode(w, op_i_return_to_trace)) { - *return_to_trace = 1; - cpp += 1; - } else if (BeamIsOpCode(w, op_i_return_time_trace)) { - cpp += 2; - } else { - break; - } + Eterm *cpp = c_p->stop; + + for (;;) { + BeamInstr w = *cp_val(*cpp); + if (BeamIsOpCode(w, op_return_trace)) { + cpp += 3; + } else if (BeamIsOpCode(w, op_i_return_to_trace)) { + *return_to_trace = 1; + cpp += 1; + } else if (BeamIsOpCode(w, op_i_return_time_trace)) { + cpp += 2; + } else { + break; } - c_p->cp = (BeamInstr *) cp_val(*cpp); - ASSERT(is_CP(*cpp)); } + c_p->stop[0] = (Eterm) cp_val(*cpp); + ASSERT(is_CP(*cpp)); } BeamInstr @@ -743,12 +704,13 @@ erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg) if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE) { Eterm w; + Eterm* E; erts_trace_time_call(c_p, info, bp->time); - w = (BeamInstr) *c_p->cp; + E = c_p->stop; + w = (BeamInstr) E[0]; if (! (BeamIsOpCode(w, op_i_return_time_trace) || BeamIsOpCode(w, op_return_trace) || BeamIsOpCode(w, op_i_return_to_trace)) ) { - Eterm* E = c_p->stop; ASSERT(c_p->htop <= E && E <= c_p->hend); if (E - 2 < c_p->htop) { (void) erts_garbage_collect(c_p, 2, reg, info->mfa.arity); @@ -759,9 +721,8 @@ erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg) ASSERT(c_p->htop <= E && E <= c_p->hend); E -= 2; - E[0] = make_cp(erts_codeinfo_to_code(info)); - E[1] = make_cp(c_p->cp); /* original return address */ - c_p->cp = beam_return_time_trace; + E[1] = make_cp(erts_codeinfo_to_code(info)); + E[0] = (Eterm) beam_return_time_trace; c_p->stop = E; } } @@ -773,237 +734,24 @@ erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg) } } -/* - * Entry point called by the trace wrap functions in erl_bif_wrap.c - * - * The trace wrap functions are themselves called through the export - * entries instead of the original BIF functions. - */ -Eterm -erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) -{ - Eterm result; - Eterm (*func)(Process*, Eterm*, BeamInstr*); - Export* ep = bif_export[bif_index]; - Uint32 flags = 0, flags_meta = 0; - ErtsTracer meta_tracer = erts_tracer_nil; - int applying = (I == ep->beam); /* Yup, the apply code for a bif - * is actually in the - * export entry */ - BeamInstr *cp = p->cp; - GenericBp* g; - GenericBpData* bp = NULL; - Uint bp_flags = 0; - int return_to_trace = 0; - - ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); - - g = ep->info.u.gen_bp; - if (g) { - bp = &g->data[erts_active_bp_ix()]; - bp_flags = bp->flags; - } - - /* - * Make continuation pointer OK, it is not during direct BIF calls, - * but it is correct during apply of bif. - */ - if (!applying) { - p->cp = I; - } else { - fixup_cp_before_trace(p, &return_to_trace); - } - if (bp_flags & (ERTS_BPF_LOCAL_TRACE|ERTS_BPF_GLOBAL_TRACE) && - IS_TRACED_FL(p, F_TRACE_CALLS)) { - int local = !!(bp_flags & ERTS_BPF_LOCAL_TRACE); - flags = erts_call_trace(p, &ep->info, bp->local_ms, args, - local, &ERTS_TRACER(p)); - } - if (bp_flags & ERTS_BPF_META_TRACE) { - ErtsTracer old_tracer; - - meta_tracer = erts_atomic_read_nob(&bp->meta_tracer->tracer); - old_tracer = meta_tracer; - flags_meta = erts_call_trace(p, &ep->info, bp->meta_ms, args, - 0, &meta_tracer); - - if (!ERTS_TRACER_COMPARE(old_tracer, meta_tracer)) { - ErtsTracer new_tracer = erts_tracer_nil; - erts_tracer_update(&new_tracer, meta_tracer); - if (old_tracer == erts_atomic_cmpxchg_acqb( - &bp->meta_tracer->tracer, - (erts_aint_t)new_tracer, - (erts_aint_t)old_tracer)) { - ERTS_TRACER_CLEAR(&old_tracer); - } else { - ERTS_TRACER_CLEAR(&new_tracer); - } - } - } - if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE && - IS_TRACED_FL(p, F_TRACE_CALLS)) { - erts_trace_time_call(p, &ep->info, bp->time); - } - - /* Restore original continuation pointer (if changed). */ - p->cp = cp; - - func = bif_table[bif_index].f; - - result = func(p, args, I); - - if (erts_nif_export_check_save_trace(p, result, - applying, ep, - cp, flags, - flags_meta, I, - meta_tracer)) { - /* - * erts_bif_trace_epilogue() will be called - * later when appropriate via the NIF export - * scheduling functionality... - */ - return result; - } - - return erts_bif_trace_epilogue(p, result, applying, ep, cp, - flags, flags_meta, I, - meta_tracer); -} - -Eterm -erts_bif_trace_epilogue(Process *p, Eterm result, int applying, - Export* ep, BeamInstr *cp, Uint32 flags, - Uint32 flags_meta, BeamInstr* I, - ErtsTracer meta_tracer) -{ - if (applying && (flags & MATCH_SET_RETURN_TO_TRACE)) { - BeamInstr i_return_trace = beam_return_trace[0]; - BeamInstr i_return_to_trace = beam_return_to_trace[0]; - BeamInstr i_return_time_trace = beam_return_time_trace[0]; - Eterm *cpp; - /* Maybe advance cp to skip trace stack frames */ - for (cpp = p->stop; ; cp = cp_val(*cpp++)) { - if (*cp == i_return_trace) { - /* Skip stack frame variables */ - while (is_not_CP(*cpp)) cpp++; - cpp += 2; /* Skip return_trace parameters */ - } else if (*cp == i_return_time_trace) { - /* Skip stack frame variables */ - while (is_not_CP(*cpp)) cpp++; - cpp += 1; /* Skip return_time_trace parameters */ - } else if (*cp == i_return_to_trace) { - /* A return_to trace message is going to be generated - * by normal means, so we do not have to. - */ - cp = NULL; - break; - } else break; - } - } - - /* Try to get these in the order - * they usually appear in normal code... */ - if (is_non_value(result)) { - Uint reason = p->freason; - if (reason != TRAP) { - Eterm class; - Eterm value = p->fvalue; - /* Expand error value like in handle_error() */ - if (reason & EXF_ARGLIST) { - Eterm *tp; - ASSERT(is_tuple(value)); - tp = tuple_val(value); - value = tp[1]; - } - if ((reason & EXF_THROWN) && (p->catches <= 0)) { - Eterm *hp = HAlloc(p, 3); - value = TUPLE2(hp, am_nocatch, value); - reason = EXC_ERROR; - } - /* Note: expand_error_value() could theoretically - * allocate on the heap, but not for any error - * returned by a BIF, and it would do no harm, - * just be annoying. - */ - value = expand_error_value(p, reason, value); - class = exception_tag[GET_EXC_CLASS(reason)]; - - if (flags_meta & MATCH_SET_EXCEPTION_TRACE) { - erts_trace_exception(p, &ep->info.mfa, class, value, - &meta_tracer); - } - if (flags & MATCH_SET_EXCEPTION_TRACE) { - erts_trace_exception(p, &ep->info.mfa, class, value, - &ERTS_TRACER(p)); - } - if ((flags & MATCH_SET_RETURN_TO_TRACE) && p->catches > 0) { - /* can only happen if(local)*/ - Eterm *ptr = p->stop; - ASSERT(is_CP(*ptr)); - ASSERT(ptr <= STACK_START(p)); - /* Search the nearest stack frame for a catch */ - while (++ptr < STACK_START(p)) { - if (is_CP(*ptr)) break; - if (is_catch(*ptr)) { - if (applying) { - /* Apply of BIF, cp is in calling function */ - if (cp) erts_trace_return_to(p, cp); - } else { - /* Direct bif call, I points into - * calling function */ - erts_trace_return_to(p, I); - } - } - } - } - if ((flags_meta|flags) & MATCH_SET_EXCEPTION_TRACE) { - erts_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); - ERTS_TRACE_FLAGS(p) |= F_EXCEPTION_TRACE; - erts_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); - } - } - } else { - if (flags_meta & MATCH_SET_RX_TRACE) { - erts_trace_return(p, &ep->info.mfa, result, &meta_tracer); - } - /* MATCH_SET_RETURN_TO_TRACE cannot occur if(meta) */ - if (flags & MATCH_SET_RX_TRACE) { - erts_trace_return(p, &ep->info.mfa, result, &ERTS_TRACER(p)); - } - if (flags & MATCH_SET_RETURN_TO_TRACE && - IS_TRACED_FL(p, F_TRACE_RETURN_TO)) { - /* can only happen if(local)*/ - if (applying) { - /* Apply of BIF, cp is in calling function */ - if (cp) erts_trace_return_to(p, cp); - } else { - /* Direct bif call, I points into calling function */ - erts_trace_return_to(p, I); - } - } - } - ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); - return result; -} - static ErtsTracer do_call_trace(Process* c_p, ErtsCodeInfo* info, Eterm* reg, int local, Binary* ms, ErtsTracer tracer) { int return_to_trace = 0; - BeamInstr *cp_save = c_p->cp; Uint32 flags; Uint need = 0; + Eterm cp_save; Eterm* E = c_p->stop; - fixup_cp_before_trace(c_p, &return_to_trace); + cp_save = E[0]; + fixup_cp_before_trace(c_p, &return_to_trace); ERTS_UNREQ_PROC_MAIN_LOCK(c_p); flags = erts_call_trace(c_p, info, ms, reg, local, &tracer); ERTS_REQ_PROC_MAIN_LOCK(c_p); - /* restore cp after potential fixup */ - c_p->cp = cp_save; + E[0] = cp_save; ASSERT(!ERTS_PROC_IS_EXITING(c_p)); if ((flags & MATCH_SET_RETURN_TO_TRACE) && !return_to_trace) { @@ -1023,28 +771,23 @@ do_call_trace(Process* c_p, ErtsCodeInfo* info, Eterm* reg, if (flags & MATCH_SET_RETURN_TO_TRACE && !return_to_trace) { E -= 1; ASSERT(c_p->htop <= E && E <= c_p->hend); - E[0] = make_cp(c_p->cp); - c_p->cp = beam_return_to_trace; + E[0] = (Eterm) beam_return_to_trace; + c_p->stop = E; } - if (flags & MATCH_SET_RX_TRACE) - { + if (flags & MATCH_SET_RX_TRACE) { E -= 3; c_p->stop = E; ASSERT(c_p->htop <= E && E <= c_p->hend); ASSERT(is_CP((Eterm) (UWord) (&info->mfa.module))); ASSERT(IS_TRACER_VALID(tracer)); - E[2] = make_cp(c_p->cp); - E[1] = copy_object(tracer, c_p); - E[0] = make_cp(&info->mfa.module); - /* We ARE at the beginning of an instruction, - the funcinfo is above i. */ - c_p->cp = (flags & MATCH_SET_EXCEPTION_TRACE) ? - beam_exception_trace : beam_return_trace; + E[2] = copy_object(tracer, c_p); + E[1] = make_cp(&info->mfa.module); + E[0] = (Eterm) ((flags & MATCH_SET_EXCEPTION_TRACE) ? + beam_exception_trace : beam_return_trace); erts_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); ERTS_TRACE_FLAGS(c_p) |= F_EXCEPTION_TRACE; erts_proc_unlock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); - } else - c_p->stop = E; + } return tracer; } diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index a64765822b..54e84e7e4f 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -119,20 +119,16 @@ void erts_bp_free_matched_functions(BpFunctions* f); void erts_install_breakpoints(BpFunctions* f); void erts_uninstall_breakpoints(BpFunctions* f); void erts_consolidate_bp_data(BpFunctions* f, int local); -void erts_consolidate_bif_bp_data(void); void erts_set_trace_break(BpFunctions *f, Binary *match_spec); void erts_clear_trace_break(BpFunctions *f); -void erts_set_call_trace_bif(ErtsCodeInfo *ci, Binary *match_spec, int local); -void erts_clear_call_trace_bif(ErtsCodeInfo *ci, int local); +void erts_set_export_trace(ErtsCodeInfo *ci, Binary *match_spec, int local); +void erts_clear_export_trace(ErtsCodeInfo *ci, int local); void erts_set_mtrace_break(BpFunctions *f, Binary *match_spec, ErtsTracer tracer); void erts_clear_mtrace_break(BpFunctions *f); -void erts_set_mtrace_bif(ErtsCodeInfo *ci, Binary *match_spec, - ErtsTracer tracer); -void erts_clear_mtrace_bif(ErtsCodeInfo *ci); void erts_set_debug_break(BpFunctions *f); void erts_clear_debug_break(BpFunctions *f); @@ -142,7 +138,7 @@ void erts_clear_count_break(BpFunctions *f); void erts_clear_all_breaks(BpFunctions* f); int erts_clear_module_break(Module *modp); -void erts_clear_export_break(Module *modp, ErtsCodeInfo* ci); +void erts_clear_export_break(Module *modp, Export *ep); BeamInstr erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *ci, Eterm* reg); BeamInstr erts_trace_break(Process *p, ErtsCodeInfo *ci, Eterm *args, @@ -151,8 +147,6 @@ BeamInstr erts_trace_break(Process *p, ErtsCodeInfo *ci, Eterm *args, int erts_is_trace_break(ErtsCodeInfo *ci, Binary **match_spec_ret, int local); int erts_is_mtrace_break(ErtsCodeInfo *ci, Binary **match_spec_ret, ErtsTracer *tracer_ret); -int erts_is_mtrace_bif(ErtsCodeInfo *ci, Binary **match_spec_ret, - ErtsTracer *tracer_ret); int erts_is_native_break(ErtsCodeInfo *ci); int erts_is_count_break(ErtsCodeInfo *ci, Uint *count_ret); int erts_is_time_break(Process *p, ErtsCodeInfo *ci, Eterm *call_time); @@ -163,10 +157,6 @@ void erts_schedule_time_break(Process *p, Uint out); void erts_set_time_break(BpFunctions *f, enum erts_break_op); void erts_clear_time_break(BpFunctions *f); -int erts_is_time_trace_bif(Process *p, ErtsCodeInfo *ci, Eterm *call_time); -void erts_set_time_trace_bif(ErtsCodeInfo *ci, enum erts_break_op); -void erts_clear_time_trace_bif(ErtsCodeInfo *ci); - ErtsCodeInfo *erts_find_local_func(ErtsCodeMFA *mfa); #if ERTS_GLB_INLINE_INCL_FUNC_DEF diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 4d52435139..6a9a6b7dc9 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -332,7 +332,7 @@ erts_debug_disassemble_1(BIF_ALIST_1) "unknown " HEXF "\n", instr); code_ptr++; } - if (i == op_call_nif) { + if (i == op_call_nif_WWW) { /* * The rest of the code will not be executed. Don't disassemble any * more code in this function. diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 07c16e3415..ee70b138e5 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -111,10 +111,9 @@ do { \ #define CHECK_ALIGNED(Dst) ASSERT((((Uint)&Dst) & (sizeof(Uint)-1)) == 0) -#define GET_BIF_MODULE(p) (p->info.mfa.module) -#define GET_BIF_FUNCTION(p) (p->info.mfa.function) -#define GET_BIF_ARITY(p) (p->info.mfa.arity) -#define GET_BIF_ADDRESS(p) ((BifFunction) (p->beam[1])) +#define GET_EXPORT_MODULE(p) ((p)->info.mfa.module) +#define GET_EXPORT_FUNCTION(p) ((p)->info.mfa.function) +#define GET_EXPORT_ARITY(p) ((p)->info.mfa.arity) #define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm)))) @@ -141,10 +140,6 @@ do { \ BeamCodeAddr(IP) < (BeamInstr)LabelAddr(end_emulator_loop)) #endif /* NO_JUMP_TABLE */ -#define SET_CP(p, ip) \ - ASSERT(VALID_INSTR(*(ip))); \ - (p)->cp = (ip) - #define SET_I(ip) \ ASSERT(VALID_INSTR(* (Eterm *)(ip))); \ I = (ip) @@ -254,72 +249,6 @@ void** beam_ops; #define Q(N) (N*sizeof(Eterm *)) #define l(N) (freg[N].fd) -/* - * Check that we haven't used the reductions and jump to function pointed to by - * the I register. If we are out of reductions, do a context switch. - */ - -#define DispatchMacro() \ - do { \ - BeamInstr dis_next; \ - dis_next = *I; \ - CHECK_ARGS(I); \ - if (FCALLS > 0 || FCALLS > neg_o_reds) { \ - FCALLS--; \ - Goto(dis_next); \ - } else { \ - goto context_switch; \ - } \ - } while (0) \ - -#define DispatchMacroFun() \ - do { \ - BeamInstr dis_next; \ - dis_next = *I; \ - CHECK_ARGS(I); \ - if (FCALLS > 0 || FCALLS > neg_o_reds) { \ - FCALLS--; \ - Goto(dis_next); \ - } else { \ - goto context_switch_fun; \ - } \ - } while (0) - -#define DispatchMacrox() \ - do { \ - if (FCALLS > 0) { \ - BeamInstr dis_next; \ - SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \ - dis_next = *I; \ - FCALLS--; \ - CHECK_ARGS(I); \ - Goto(dis_next); \ - } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p) \ - && FCALLS > neg_o_reds) { \ - goto save_calls1; \ - } else { \ - SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \ - CHECK_ARGS(I); \ - goto context_switch; \ - } \ - } while (0) - -#ifdef DEBUG -/* - * To simplify breakpoint setting, put the code in one place only and jump to it. - */ -# define Dispatch() goto do_dispatch -# define Dispatchx() goto do_dispatchx -# define Dispatchfun() goto do_dispatchfun -#else -/* - * Inline for speed. - */ -# define Dispatch() DispatchMacro() -# define Dispatchx() DispatchMacrox() -# define Dispatchfun() DispatchMacroFun() -#endif - #define Arg(N) I[(N)+1] #define GetSource(raw, dst) \ @@ -352,19 +281,6 @@ do { \ } \ } while(0) -#define DispatchReturn \ -do { \ - if (FCALLS > 0 || FCALLS > neg_o_reds) { \ - FCALLS--; \ - Goto(*I); \ - } \ - else { \ - c_p->current = NULL; \ - c_p->arity = 1; \ - goto context_switch3; \ - } \ -} while (0) - #ifdef DEBUG /* Better static type testing by the C compiler */ # define BEAM_IS_TUPLE(Src) is_tuple(Src) @@ -521,10 +437,10 @@ init_emulator(void) } \ } while(0) -#define DTRACE_RETURN_FROM_PC(p) \ +#define DTRACE_RETURN_FROM_PC(p, i) \ do { \ ErtsCodeMFA* cmfa; \ - if (DTRACE_ENABLED(function_return) && (cmfa = find_function_from_pc((p)->cp))) { \ + if (DTRACE_ENABLED(function_return) && (cmfa = find_function_from_pc(i))) { \ DTRACE_RETURN((p), cmfa); \ } \ } while(0) @@ -534,7 +450,7 @@ init_emulator(void) #define DTRACE_GLOBAL_CALL(p, mfa) do {} while (0) #define DTRACE_GLOBAL_CALL_FROM_EXPORT(p, e) do {} while (0) #define DTRACE_RETURN(p, mfa) do {} while (0) -#define DTRACE_RETURN_FROM_PC(p) do {} while (0) +#define DTRACE_RETURN_FROM_PC(p, i) do {} while (0) #define DTRACE_BIF_ENTRY(p, mfa) do {} while (0) #define DTRACE_BIF_RETURN(p, mfa) do {} while (0) #define DTRACE_NIF_ENTRY(p, mfa) do {} while (0) @@ -772,27 +688,9 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) #endif #include "beam_hot.h" - -#ifdef DEBUG /* - * Set a breakpoint here to get control just after a call instruction. - * I points to the first instruction in the called function. - * - * In gdb, use 'call dis(I-5, 1)' to show the name of the function. - */ - do_dispatch: - DispatchMacro(); - - do_dispatchx: - DispatchMacrox(); - - do_dispatchfun: - DispatchMacroFun(); - -#endif - - /* - * Jumped to from the Dispatch() macro when the reductions are used up. + * The labels are jumped to from the $DISPATCH() macros when the reductions + * are used up. * * Since the I register points just beyond the FuncBegin instruction, we * can get the module, function, and arity for the function being @@ -986,18 +884,42 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) } #endif return; /* Never executed */ +} - save_calls1: - { - BeamInstr dis_next; +/* + * Enter all BIFs into the export table. + * + * Note that they will all call the error_handler until their modules have been + * loaded, which may prevent the system from booting if BIFs from non-preloaded + * modules are apply/3'd while loading code. Ordinary BIF calls will work fine + * however since they won't go through export entries. + */ +static void install_bifs(void) { + int i; + + for (i = 0; i < BIF_SIZE; i++) { + BifEntry *entry; + Export *ep; + int j; - save_calls(c_p, (Export *) Arg(0)); + entry = &bif_table[i]; - SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); + ep = erts_export_put(entry->module, entry->name, entry->arity); - dis_next = *I; - FCALLS--; - Goto(dis_next); + ep->info.op = BeamOpCodeAddr(op_i_func_info_IaaI); + ep->info.mfa.module = entry->module; + ep->info.mfa.function = entry->name; + ep->info.mfa.arity = entry->arity; + ep->bif_table_index = i; + + memset(&ep->trampoline, 0, sizeof(ep->trampoline)); + ep->trampoline.op = BeamOpCodeAddr(op_call_error_handler); + + for (j = 0; j < ERTS_NUM_CODE_IX; j++) { + ep->addressv[j] = ep->trampoline.raw; + } + + bif_export[i] = ep; } } @@ -1008,43 +930,30 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) static void init_emulator_finish(void) { - int i; - Export* ep; - #if defined(ARCH_64) && defined(CODE_MODEL_SMALL) - for (i = 0; i < NUMBER_OF_OPCODES; i++) { - BeamInstr instr = BeamOpCodeAddr(i); - if (instr >= (1ull << 32)) { - erts_exit(ERTS_ERROR_EXIT, - "This run-time was supposed be compiled with all code below 2Gb,\n" - "but the instruction '%s' is located at %016lx.\n", - opc[i].name, instr); - } - } + int i; + + for (i = 0; i < NUMBER_OF_OPCODES; i++) { + BeamInstr instr = BeamOpCodeAddr(i); + if (instr >= (1ull << 32)) { + erts_exit(ERTS_ERROR_EXIT, + "This run-time was supposed be compiled with all code below 2Gb,\n" + "but the instruction '%s' is located at %016lx.\n", + opc[i].name, instr); + } + } #endif - beam_apply[0] = BeamOpCodeAddr(op_i_apply); - beam_apply[1] = BeamOpCodeAddr(op_normal_exit); - beam_exit[0] = BeamOpCodeAddr(op_error_action_code); - beam_continue_exit[0] = BeamOpCodeAddr(op_continue_exit); - beam_return_to_trace[0] = BeamOpCodeAddr(op_i_return_to_trace); - beam_return_trace[0] = BeamOpCodeAddr(op_return_trace); - beam_exception_trace[0] = BeamOpCodeAddr(op_return_trace); /* UGLY */ - beam_return_time_trace[0] = BeamOpCodeAddr(op_i_return_time_trace); + beam_apply[0] = BeamOpCodeAddr(op_i_apply); + beam_apply[1] = BeamOpCodeAddr(op_normal_exit); + beam_exit[0] = BeamOpCodeAddr(op_error_action_code); + beam_continue_exit[0] = BeamOpCodeAddr(op_continue_exit); + beam_return_to_trace[0] = BeamOpCodeAddr(op_i_return_to_trace); + beam_return_trace[0] = BeamOpCodeAddr(op_return_trace); + beam_exception_trace[0] = BeamOpCodeAddr(op_return_trace); /* UGLY */ + beam_return_time_trace[0] = BeamOpCodeAddr(op_i_return_time_trace); - /* - * Enter all BIFs into the export table. - */ - for (i = 0; i < BIF_SIZE; i++) { - ep = erts_export_put(bif_table[i].module, - bif_table[i].name, - bif_table[i].arity); - bif_export[i] = ep; - ep->beam[0] = BeamOpCodeAddr(op_apply_bif); - ep->beam[1] = (BeamInstr) bif_table[i].f; - /* XXX: set func info for bifs */ - ep->info.op = BeamOpCodeAddr(op_i_func_info_IaaI); - } + install_bifs(); } /* @@ -1257,7 +1166,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) * I[2]: Pointer to erl_module_nif * I[3]: Function pointer to dirty NIF * - * This layout is determined by the NifExport struct + * This layout is determined by the ErtsNativeFunc struct */ ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); @@ -1271,11 +1180,11 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) ERTS_UNREQ_PROC_MAIN_LOCK(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - if (BeamIsOpCode(*I, op_apply_bif)) { + if (BeamIsOpCode(*I, op_call_bif_W)) { exiting = erts_call_dirty_bif(esdp, c_p, I, reg); } else { - ASSERT(BeamIsOpCode(*I, op_call_nif)); + ASSERT(BeamIsOpCode(*I, op_call_nif_WWW)); exiting = erts_call_dirty_nif(esdp, c_p, I, reg); } @@ -1344,6 +1253,33 @@ Eterm error_atom[NUMBER_EXIT_CODES] = { am_badkey, /* 19 */ }; +/* Returns the return address at E[0] in printable form, skipping tracing in + * the same manner as gather_stacktrace. + * + * This is needed to generate correct stacktraces when throwing errors from + * instructions that return like an ordinary function, such as call_nif. */ +BeamInstr *erts_printable_return_address(Process* p, Eterm *E) { + Eterm *ptr = E; + + ASSERT(is_CP(*ptr)); + + while (ptr < STACK_START(p)) { + BeamInstr *cp = cp_val(*ptr); + + if (cp == beam_exception_trace || cp == beam_return_trace) { + ptr += 3; + } else if (cp == beam_return_time_trace) { + ptr += 2; + } else if (cp == beam_return_to_trace) { + ptr += 1; + } else { + return cp; + } + } + + ERTS_ASSERT(!"No continuation pointer on stack"); +} + /* * To fully understand the error handling, one must keep in mind that * when an exception is thrown, the search for a handler can jump back @@ -1373,14 +1309,14 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, ErtsCodeMFA *bif_mfa) ASSERT(c_p->freason != TRAP); /* Should have been handled earlier. */ - if (c_p->freason & EXF_RESTORE_NIF) - erts_nif_export_restore_error(c_p, &pc, reg, &bif_mfa); + if (c_p->freason & EXF_RESTORE_NFUNC) + erts_nfunc_restore_error(c_p, &pc, reg, &bif_mfa); #ifdef DEBUG if (bif_mfa) { - /* Verify that bif_mfa does not point into our nif export */ - NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); - ASSERT(!nep || !ErtsInArea(bif_mfa, (char *)nep, sizeof(NifExport))); + /* Verify that bif_mfa does not point into our native function wrapper */ + ErtsNativeFunc *nep = ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(c_p); + ASSERT(!nep || !ErtsInArea(bif_mfa, (char *)nep, sizeof(ErtsNativeFunc))); } #endif @@ -1443,7 +1379,7 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, ErtsCodeMFA *bif_mfa) reg[2] = Value; reg[3] = c_p->ftrace; if ((new_pc = next_catch(c_p, reg))) { - c_p->cp = 0; /* To avoid keeping stale references. */ + c_p->stop[0] = NIL; /* To avoid keeping stale references. */ ERTS_RECV_MARK_CLEAR(c_p); /* No longer safe to use this position */ return new_pc; } @@ -1481,35 +1417,6 @@ next_catch(Process* c_p, Eterm *reg) { return NULL; } - /* - * Better safe than sorry here. In debug builds, produce a core - * dump if the top of the stack doesn't point to a continuation - * pointer. In other builds, ignore a non-CP at the top of stack. - */ - ASSERT(is_CP(*ptr)); - if ((is_not_CP(*ptr) || (*cp_val(*ptr) != i_return_trace && - *cp_val(*ptr) != i_return_to_trace && - *cp_val(*ptr) != i_return_time_trace )) - && c_p->cp) { - /* Can not follow cp here - code may be unloaded */ - BeamInstr *cpp = c_p->cp; - if (cpp == beam_exception_trace) { - ErtsCodeMFA *mfa = (ErtsCodeMFA*)cp_val(ptr[0]); - erts_trace_exception(c_p, mfa, - reg[1], reg[2], - ERTS_TRACER_FROM_ETERM(ptr+1)); - /* Skip return_trace parameters */ - ptr += 2; - } else if (cpp == beam_return_trace) { - /* Skip return_trace parameters */ - ptr += 2; - } else if (cpp == beam_return_time_trace) { - /* Skip return_trace parameters */ - ptr += 1; - } else if (cpp == beam_return_to_trace) { - have_return_to_trace = !0; /* Record next cp */ - } - } while (ptr < STACK_START(c_p)) { if (is_catch(*ptr)) { if (active_catches) goto found_catch; @@ -1664,6 +1571,54 @@ expand_error_value(Process* c_p, Uint freason, Eterm Value) { return Value; } + +static void +gather_stacktrace(Process* p, struct StackTrace* s, int depth) +{ + BeamInstr *prev; + Eterm *ptr; + + if (depth == 0) { + return; + } + + prev = s->depth ? s->trace[s->depth - 1] : s->pc; + ptr = p->stop; + + /* + * Traverse the stack backwards and add all unique continuation + * pointers to the buffer, up to the maximum stack trace size. + * + * Skip trace stack frames. + */ + + ASSERT(ptr >= STACK_TOP(p) && ptr <= STACK_START(p)); + + while (ptr < STACK_START(p) && depth > 0) { + if (is_CP(*ptr)) { + BeamInstr *cp = cp_val(*ptr); + + if (cp == beam_exception_trace || cp == beam_return_trace) { + ptr += 3; + } else if (cp == beam_return_time_trace) { + ptr += 2; + } else if (cp == beam_return_to_trace) { + ptr += 1; + } else { + if (cp != prev) { + /* Record non-duplicates only */ + prev = cp; + s->trace[s->depth++] = cp - 1; + depth--; + } + ptr++; + } + } else { + ptr++; + } + } +} + /* * Quick-saving the stack trace in an internal form on the heap. Note * that c_p->ftrace will point to a cons cell which holds the given args @@ -1750,11 +1705,6 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, s->trace[s->depth++] = pc; depth--; } - /* Save second stack entry if CP is valid and different from pc */ - if (depth > 0 && c_p->cp != 0 && c_p->cp != pc) { - s->trace[s->depth++] = c_p->cp - 1; - depth--; - } s->pc = NULL; args = make_arglist(c_p, reg, bif_mfa->arity); /* Overwrite CAR(c_p->ftrace) */ } else { @@ -1762,9 +1712,9 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, non_bif_stacktrace: s->current = c_p->current; - /* + /* * For a function_clause error, the arguments are in the beam - * registers, c_p->cp is valid, and c_p->current is set. + * registers and c_p->current is set. */ if ( (GET_EXC_INDEX(s->freason)) == (GET_EXC_INDEX(EXC_FUNCTION_CLAUSE)) ) { @@ -1772,18 +1722,8 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, ASSERT(s->current); a = s->current->arity; args = make_arglist(c_p, reg, a); /* Overwrite CAR(c_p->ftrace) */ - /* Save first stack entry */ - ASSERT(c_p->cp); - if (depth > 0) { - s->trace[s->depth++] = c_p->cp - 1; - depth--; - } s->pc = NULL; /* Ignore pc */ } else { - if (depth > 0 && c_p->cp != 0 && c_p->cp != pc) { - s->trace[s->depth++] = c_p->cp - 1; - depth--; - } s->pc = pc; } } @@ -1796,80 +1736,13 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, } /* Save the actual stack trace */ - erts_save_stacktrace(c_p, s, depth); + gather_stacktrace(c_p, s, depth); } void erts_save_stacktrace(Process* p, struct StackTrace* s, int depth) { - if (depth > 0) { - Eterm *ptr; - BeamInstr *prev = s->depth ? s->trace[s->depth-1] : NULL; - BeamInstr i_return_trace = beam_return_trace[0]; - BeamInstr i_return_to_trace = beam_return_to_trace[0]; - - /* - * Traverse the stack backwards and add all unique continuation - * pointers to the buffer, up to the maximum stack trace size. - * - * Skip trace stack frames. - */ - ptr = p->stop; - if (ptr < STACK_START(p) && - (is_not_CP(*ptr)|| (*cp_val(*ptr) != i_return_trace && - *cp_val(*ptr) != i_return_to_trace)) && - p->cp) { - /* Cannot follow cp here - code may be unloaded */ - BeamInstr *cpp = p->cp; - int trace_cp; - if (cpp == beam_exception_trace || cpp == beam_return_trace) { - /* Skip return_trace parameters */ - ptr += 2; - trace_cp = 1; - } else if (cpp == beam_return_to_trace) { - /* Skip return_to_trace parameters */ - ptr += 1; - trace_cp = 1; - } - else { - trace_cp = 0; - } - if (trace_cp && s->pc == cpp) { - /* - * If process 'cp' points to a return/exception trace - * instruction and 'cp' has been saved as 'pc' in - * stacktrace, we need to update 'pc' in stacktrace - * with the actual 'cp' located on the top of the - * stack; otherwise, we will lose the top stackframe - * when building the stack trace. - */ - ASSERT(is_CP(p->stop[0])); - s->pc = cp_val(p->stop[0]); - } - } - while (ptr < STACK_START(p) && depth > 0) { - if (is_CP(*ptr)) { - if (*cp_val(*ptr) == i_return_trace) { - /* Skip stack frame variables */ - do ++ptr; while (is_not_CP(*ptr)); - /* Skip return_trace parameters */ - ptr += 2; - } else if (*cp_val(*ptr) == i_return_to_trace) { - /* Skip stack frame variables */ - do ++ptr; while (is_not_CP(*ptr)); - } else { - BeamInstr *cp = cp_val(*ptr); - if (cp != prev) { - /* Record non-duplicates only */ - prev = cp; - s->trace[s->depth++] = cp - 1; - depth--; - } - ptr++; - } - } else ptr++; - } - } + gather_stacktrace(p, s, depth); } /* @@ -2128,95 +2001,66 @@ apply_bif_error_adjustment(Process *p, Export *ep, Eterm *reg, Uint arity, BeamInstr *I, Uint stack_offset) { + int apply_only; + Uint need; + + need = stack_offset /* bytes */ / sizeof(Eterm); + apply_only = stack_offset == 0; + /* * I is only set when the apply is a tail call, i.e., * from the instructions i_apply_only, i_apply_last_P, * and apply_last_IP. */ - if (I - && BeamIsOpCode(ep->beam[0], op_apply_bif) - && (ep == bif_export[BIF_error_1] - || ep == bif_export[BIF_error_2] - || ep == bif_export[BIF_exit_1] - || ep == bif_export[BIF_throw_1])) { - /* - * We are about to tail apply one of the BIFs - * erlang:error/1, erlang:error/2, erlang:exit/1, - * or erlang:throw/1. Error handling of these BIFs is - * special! - * - * We need 'p->cp' to point into the calling - * function when handling the error after the BIF has - * been applied. This in order to get the topmost - * stackframe correct. Without the following adjustment, - * 'p->cp' will point into the function that called - * current function when handling the error. We add a - * dummy stackframe in order to achieve this. - * - * Note that these BIFs unconditionally will cause - * an exception to be raised. That is, our modifications - * of 'p->cp' as well as the stack will be corrected by - * the error handling code. - * - * If we find an exception/return-to trace continuation - * pointer as the topmost continuation pointer, we do not - * need to do anything since the information already will - * be available for generation of the stacktrace. - */ - int apply_only = stack_offset == 0; - BeamInstr *cpp; + if (!(I && (ep == bif_export[BIF_error_1] || + ep == bif_export[BIF_error_2] || + ep == bif_export[BIF_exit_1] || + ep == bif_export[BIF_throw_1]))) { + return; + } - if (apply_only) { - ASSERT(p->cp != NULL); - cpp = p->cp; - } - else { - ASSERT(is_CP(p->stop[0])); - cpp = cp_val(p->stop[0]); - } + /* + * We are about to tail apply one of the BIFs erlang:error/1, + * erlang:error/2, erlang:exit/1, or erlang:throw/1. Error handling of + * these BIFs is special! + * + * We need the topmost continuation pointer to point into the calling + * function when handling the error after the BIF has been applied. This in + * order to get the topmost stackframe correct. + * + * Note that these BIFs will unconditionally cause an exception to be + * raised. That is, our modifications of the stack will be corrected by the + * error handling code. + */ + if (need == 0) { + need = 1; /* i_apply_only */ + } - if (cpp != beam_exception_trace - && cpp != beam_return_trace - && cpp != beam_return_to_trace) { - Uint need = stack_offset /* bytes */ / sizeof(Eterm); - if (need == 0) - need = 1; /* i_apply_only */ - if (p->stop - p->htop < need) - erts_garbage_collect(p, (int) need, reg, arity+1); - p->stop -= need; - - if (apply_only) { - /* - * Called from the i_apply_only instruction. - * - * 'p->cp' contains continuation pointer pointing - * into the function that called current function. - * We push that continuation pointer onto the stack, - * and set 'p->cp' to point into current function. - */ + if (p->stop - p->htop < need) { + erts_garbage_collect(p, (int) need, reg, arity+1); + } - p->stop[0] = make_cp(p->cp); - p->cp = I; - } - else { - /* - * Called from an i_apply_last_p, or apply_last_IP, - * instruction. - * - * Calling instruction will after we return read - * a continuation pointer from the stack and write - * it to 'p->cp', and then remove the topmost - * stackframe of size 'stack_offset'. - * - * We have sized the dummy-stackframe so that it - * will be removed by the instruction we currently - * are executing, and leave the stackframe that - * normally would have been removed intact. - * - */ - p->stop[0] = make_cp(I); - } - } + if (apply_only) { + /* + * Called from the i_apply_only instruction. + * + * Push the continuation pointer for the current function to the stack. + */ + p->stop -= need; + p->stop[0] = make_cp(I); + } else { + /* + * Called from an i_apply_last_* instruction. + * + * The calling instruction will deallocate a stack frame of size + * 'stack_offset'. + * + * Push the continuation pointer for the current function to the stack, + * and then add a dummy stackframe for the i_apply_last* instruction + * to discard. + */ + p->stop[0] = make_cp(I); + p->stop -= need; } } @@ -2437,10 +2281,10 @@ erts_hibernate(Process* c_p, Eterm* reg) c_p->arg_reg[0] = module; c_p->arg_reg[1] = function; c_p->arg_reg[2] = args; - c_p->stop = STACK_START(c_p); + c_p->stop = c_p->hend - 1; /* Keep first continuation pointer */ + ASSERT(c_p->stop[0] == make_cp(beam_apply+1)); c_p->catches = 0; c_p->i = beam_apply; - c_p->cp = (BeamInstr *) beam_apply+1; /* * If there are no waiting messages, garbage collect and @@ -3268,10 +3112,10 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity) e.info.mfa.arity = arity; if ((ep = export_get(&e)) == NULL) { - return 0; + return 0; } - return ep->addressv[erts_active_code_ix()] == ep->beam && - BeamIsOpCode(ep->beam[0], op_apply_bif); + + return ep->bif_table_index != -1; } diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 35f2ea6688..8fe046095f 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -141,7 +141,7 @@ typedef struct { * eventually patch with a pointer into * the export entry. */ - BifFunction bf; /* Pointer to BIF function if BIF; + Export *bif; /* Pointer to export entry if BIF; * NULL otherwise. */ } ImportEntry; @@ -315,6 +315,7 @@ typedef struct LoaderState { * (or 0 if there is no on_load function) */ int otp_20_or_higher; /* Compiled with OTP 20 or higher */ + unsigned max_opcode; /* Highest opcode used in module */ /* * Atom table. @@ -844,17 +845,23 @@ erts_finish_loading(Binary* magic, Process* c_p, if (ep == NULL || ep->info.mfa.module != module) { continue; } - if (ep->addressv[code_ix] == ep->beam) { - if (BeamIsOpCode(ep->beam[0], op_apply_bif)) { - continue; - } else if (BeamIsOpCode(ep->beam[0], op_i_generic_breakpoint)) { + + DBG_CHECK_EXPORT(ep, code_ix); + + if (ep->addressv[code_ix] == ep->trampoline.raw) { + if (BeamIsOpCode(ep->trampoline.op, op_i_generic_breakpoint)) { ERTS_LC_ASSERT(erts_thr_progress_is_blocking()); ASSERT(mod_tab_p->curr.num_traced_exports > 0); - erts_clear_export_break(mod_tab_p, &ep->info); - ep->addressv[code_ix] = (BeamInstr *) ep->beam[1]; - ep->beam[1] = 0; + + erts_clear_export_break(mod_tab_p, ep); + + ep->addressv[code_ix] = + (BeamInstr*)ep->trampoline.breakpoint.address; + ep->trampoline.breakpoint.address = 0; + + ASSERT(ep->addressv[code_ix] != ep->trampoline.raw); } - ASSERT(ep->beam[1] == 0); + ASSERT(ep->trampoline.breakpoint.address == 0); } } ASSERT(mod_tab_p->curr.num_breakpoints == 0); @@ -1470,15 +1477,14 @@ load_import_table(LoaderState* stp) } stp->import[i].arity = arity; stp->import[i].patches = 0; - stp->import[i].bf = NULL; + stp->import[i].bif = NULL; /* - * If the export entry refers to a BIF, get the pointer to - * the BIF function. + * If the export entry refers to a BIF, save a pointer to the BIF entry. */ if ((e = erts_active_export_entry(mod, func, arity)) != NULL) { - if (BeamIsOpCode(e->beam[0], op_apply_bif)) { - stp->import[i].bf = (BifFunction) e->beam[1]; + if (e->bif_table_index != -1) { + stp->import[i].bif = e; if (func == am_load_nif && mod == am_erlang && arity == 2) { stp->may_load_nif = 1; } @@ -1529,33 +1535,6 @@ read_export_table(LoaderState* stp) LoadError2(stp, "export table entry %u: label %u not resolved", i, n); } stp->export[i].address = address = stp->codev + value; - - /* - * Find out if there is a BIF with the same name. - */ - - if (!is_bif(stp->module, func, arity)) { - continue; - } - - /* - * This is a stub for a BIF. - * - * It should not be exported, and the information in its - * func_info instruction should be invalidated so that it - * can be filtered out by module_info(functions) and by - * any other functions that walk through all local functions. - */ - - if (stp->labels[n].num_patches > 0) { - LoadError3(stp, "there are local calls to the stub for " - "the BIF %T:%T/%d", - stp->module, func, arity); - } - stp->export[i].address = NULL; - address[-1] = 0; - address[-2] = NIL; - address[-3] = NIL; } return 1; @@ -1563,31 +1542,33 @@ read_export_table(LoaderState* stp) return 0; } - static int is_bif(Eterm mod, Eterm func, unsigned arity) { - Export* e = erts_active_export_entry(mod, func, arity); - if (e == NULL) { - return 0; - } - if (! BeamIsOpCode(e->beam[0], op_apply_bif)) { - return 0; - } - if (mod == am_erlang && func == am_apply && arity == 3) { - /* - * erlang:apply/3 is a special case -- it is implemented - * as an instruction and it is OK to redefine it. - */ - return 0; + Export *e = erts_active_export_entry(mod, func, arity); + + if (e != NULL) { + return e->bif_table_index != -1; } - return 1; + + return 0; } static int read_lambda_table(LoaderState* stp) { unsigned int i; + unsigned int otp_22_or_lower; + + /* + * Determine whether this module was compiled with OTP 22 or lower + * by looking at the max opcode number. The compiler in OTP 23 will + * always set the max opcode to the opcode for `swap` (whether + * actually used or not) so that a module compiled for OTP 23 + * cannot be loaded in earlier versions. + */ + + otp_22_or_lower = stp->max_opcode < genop_swap_2; GetInt(stp, 4, stp->num_lambdas); if (stp->num_lambdas > stp->lambdas_allocated) { @@ -1619,6 +1600,29 @@ read_lambda_table(LoaderState* stp) GetInt(stp, 4, Index); GetInt(stp, 4, stp->lambdas[i].num_free); GetInt(stp, 4, OldUniq); + + /* + * Fun entries are now keyed by the explicit ("new") index in + * the fun entry. That allows multiple make_fun2 instructions + * to share the same fun entry (when the `fun F/A` syntax is + * used). Before OTP 23, fun entries were keyed by the old + * index, which is the order of the entries in the fun + * chunk. Each make_fun2 needed to refer to its own fun entry. + * + * Modules compiled before OTP 23 can safely be loaded if the + * old index and the new index are equal. That is true for all + * modules compiled with OTP R15 and later. + */ + if (otp_22_or_lower && i != Index) { + /* + * Compiled with a compiler before OTP R15B. The new indices + * are not reliable, so it is not safe to load this module. + */ + LoadError2(stp, "please re-compile this module with an " + ERLANG_OTP_RELEASE " compiler " + "(old-style fun with indices: %d/%d)", + i, Index); + } fe = erts_put_fun_entry2(stp->module, OldUniq, i, stp->mod_md5, Index, arity-stp->lambdas[i].num_free); stp->lambdas[i].fe = fe; @@ -1839,7 +1843,6 @@ read_code_header(LoaderState* stp) { unsigned head_size; unsigned version; - unsigned opcode_max; int i; /* @@ -1871,8 +1874,8 @@ read_code_header(LoaderState* stp) /* * Verify the number of the highest opcode used. */ - GetInt(stp, 4, opcode_max); - if (opcode_max > MAX_GENERIC_OPCODE) { + GetInt(stp, 4, stp->max_opcode); + if (stp->max_opcode > MAX_GENERIC_OPCODE) { LoadError2(stp, "This BEAM file was compiled for a later version" " of the run-time system than " ERLANG_OTP_RELEASE ".\n" @@ -1880,7 +1883,7 @@ read_code_header(LoaderState* stp) ERLANG_OTP_RELEASE " compiler.\n" " (Use of opcode %d; this emulator supports " "only up to %d.)", - opcode_max, MAX_GENERIC_OPCODE); + stp->max_opcode, MAX_GENERIC_OPCODE); } GetInt(stp, 4, stp->num_labels); @@ -2500,10 +2503,14 @@ load_code(LoaderState* stp) if (i >= stp->num_imports) { LoadError1(stp, "invalid import table index %d", i); } - if (stp->import[i].bf == NULL) { + if (stp->import[i].bif == NULL) { LoadError1(stp, "not a BIF: import table index %d", i); } - code[ci++] = (BeamInstr) stp->import[i].bf; + { + int bif_index = stp->import[i].bif->bif_table_index; + BifEntry *bif_entry = &bif_table[bif_index]; + code[ci++] = (BeamInstr) bif_entry->f; + } break; case 'P': /* Byte offset into tuple or stack */ case 'Q': /* Like 'P', but packable */ @@ -2811,18 +2818,43 @@ load_code(LoaderState* stp) switch (stp->specific_op) { case op_i_func_info_IaaI: { + int padding_required; Sint offset; + if (function_number >= stp->num_functions) { LoadError1(stp, "too many functions in module (header said %u)", stp->num_functions); } - if (stp->may_load_nif) { + /* Native function calls may be larger than their stubs, so + * we'll need to make sure any potentially-native function stub + * is padded with enough room. + * + * Note that the padding is applied for the previous function, + * not the current one, so we check whether the old F/A is + * a BIF. */ + padding_required = last_func_start && (stp->may_load_nif || + is_bif(stp->module, stp->function, stp->arity)); + + /* + * Save context for error messages. + */ + stp->function = code[ci-2]; + stp->arity = code[ci-1]; + + /* + * Save current offset of into the line instruction array. + */ + if (stp->func_line) { + stp->func_line[function_number] = stp->current_li; + } + + if (padding_required) { const int finfo_ix = ci - FUNC_INFO_SZ; - if (finfo_ix - last_func_start < BEAM_NIF_MIN_FUNC_SZ && last_func_start) { + if (finfo_ix - last_func_start < BEAM_NATIVE_MIN_FUNC_SZ) { /* Must make room for call_nif op */ - int pad = BEAM_NIF_MIN_FUNC_SZ - (finfo_ix - last_func_start); - ASSERT(pad > 0 && pad < BEAM_NIF_MIN_FUNC_SZ); + int pad = BEAM_NATIVE_MIN_FUNC_SZ - (finfo_ix - last_func_start); + ASSERT(pad > 0 && pad < BEAM_NATIVE_MIN_FUNC_SZ); CodeNeed(pad); sys_memmove(&code[finfo_ix+pad], &code[finfo_ix], FUNC_INFO_SZ*sizeof(BeamInstr)); @@ -2833,20 +2865,6 @@ load_code(LoaderState* stp) } last_func_start = ci; - /* - * Save current offset of into the line instruction array. - */ - - if (stp->func_line) { - stp->func_line[function_number] = stp->current_li; - } - - /* - * Save context for error messages. - */ - stp->function = code[ci-2]; - stp->arity = code[ci-1]; - /* When this assert is triggered, it is normally a sign that the size of the ops.tab i_func_info instruction is not the same as FUNC_INFO_SZ */ @@ -2876,7 +2894,6 @@ load_code(LoaderState* stp) case op_i_bs_match_string_yfWW: new_string_patch(stp, ci-1); break; - case op_catch_yf: /* code[ci-3] &&lb_catch_yf * code[ci-2] y-register offset in E @@ -3131,27 +3148,6 @@ mixed_types(LoaderState* stp, GenOpArg Size, GenOpArg* Rest) return 0; } -static int -is_killed_apply(LoaderState* stp, GenOpArg Reg, GenOpArg Live) -{ - return Reg.type == TAG_x && Live.type == TAG_u && - Live.val+2 <= Reg.val; -} - -static int -is_killed(LoaderState* stp, GenOpArg Reg, GenOpArg Live) -{ - return Reg.type == TAG_x && Live.type == TAG_u && - Live.val <= Reg.val; -} - -static int -is_killed_by_call_fun(LoaderState* stp, GenOpArg Reg, GenOpArg Live) -{ - return Reg.type == TAG_x && Live.type == TAG_u && - Live.val+1 <= Reg.val; -} - /* * Test whether register Reg is killed by make_fun instruction that * creates the fun given by index idx. @@ -3171,14 +3167,24 @@ is_killed_by_make_fun(LoaderState* stp, GenOpArg Reg, GenOpArg idx) } } -/* - * Test whether register Reg is killed by the send instruction that follows. - */ - +/* Test whether Bif is "heavy" and should always go through its export entry */ static int -is_killed_by_send(LoaderState* stp, GenOpArg Reg) +is_heavy_bif(LoaderState* stp, GenOpArg Bif) { - return Reg.type == TAG_x && 2 <= Reg.val; + Export *bif_export; + + if (Bif.type != TAG_u || Bif.val >= stp->num_imports) { + return 0; + } + + bif_export = stp->import[Bif.val].bif; + + if (bif_export) { + int bif_index = bif_export->bif_table_index; + return bif_table[bif_index].kind == BIF_KIND_HEAVY; + } + + return 0; } /* @@ -5208,27 +5214,52 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p) */ for (i = 0; i < stp->num_exps; i++) { - Export* ep; - BeamInstr* address = stp->export[i].address; + Export* ep; + BeamInstr* address = stp->export[i].address; - if (address == NULL) { - /* Skip stub for a BIF */ - continue; - } - ep = erts_export_put(stp->module, stp->export[i].function, - stp->export[i].arity); - if (on_load) { - /* - * on_load: Don't make any of the exported functions - * callable yet. Keep any function in the current - * code callable. - */ - ep->beam[1] = (BeamInstr) address; - } - else + ep = erts_export_put(stp->module, + stp->export[i].function, + stp->export[i].arity); + + /* Fill in BIF stubs with a proper call to said BIF. */ + if (ep->bif_table_index != -1) { + erts_write_bif_wrapper(ep, address); + } + + if (on_load) { + /* + * on_load: Don't make any of the exported functions + * callable yet. Keep any function in the current + * code callable. + */ + ep->trampoline.not_loaded.deferred = (BeamInstr) address; + } else { ep->addressv[erts_staging_code_ix()] = address; + } } +#ifdef DEBUG + /* Ensure that we've loaded stubs for all BIFs in this module. */ + for (i = 0; i < BIF_SIZE; i++) { + BifEntry *entry = &bif_table[i]; + + if (stp->module == entry->module) { + Export *ep = erts_export_put(entry->module, + entry->name, + entry->arity); + BeamInstr *addr = ep->addressv[erts_staging_code_ix()]; + + if (!ErtsInArea(addr, stp->codev, stp->ci * sizeof(BeamInstr))) { + erts_exit(ERTS_ABORT_EXIT, + "Module %T doesn't export BIF %T/%i\n", + entry->module, + entry->name, + entry->arity); + } + } + } +#endif + /* * Import functions and patch all callers. */ @@ -5400,15 +5431,14 @@ transform_engine(LoaderState* st) i = instr->a[ap].val; ASSERT(i < st->num_imports); - if (i >= st->num_imports || st->import[i].bf == NULL) + if (i >= st->num_imports || st->import[i].bif == NULL) goto restart; if (bif_number != -1 && - bif_export[bif_number]->beam[1] != (BeamInstr) st->import[i].bf) { + bif_export[bif_number] != st->import[i].bif) { goto restart; } } break; - #endif #if defined(TOP_is_not_bif) case TOP_is_not_bif: @@ -5438,7 +5468,7 @@ transform_engine(LoaderState* st) * they are special. */ if (i < st->num_imports) { - if (st->import[i].bf != NULL || + if (st->import[i].bif != NULL || (st->import[i].module == am_erlang && st->import[i].function == am_apply && (st->import[i].arity == 2 || st->import[i].arity == 3))) { @@ -6283,12 +6313,12 @@ exported_from_module(Process* p, /* Process whose heap to use. */ if (ep->info.mfa.module == mod) { Eterm tuple; - - if (ep->addressv[code_ix] == ep->beam && - BeamIsOpCode(ep->beam[0], op_call_error_handler)) { - /* There is a call to the function, but it does not exist. */ - continue; - } + + if (ep->addressv[code_ix] == ep->trampoline.raw && + BeamIsOpCode(ep->trampoline.op, op_call_error_handler)) { + /* There is a call to the function, but it does not exist. */ + continue; + } if (hp == hend) { int need = 10 * 5; diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index 156c3c45e2..e7127c5b08 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -106,7 +106,7 @@ typedef struct beam_code_header { }BeamCodeHeader; -# define BEAM_NIF_MIN_FUNC_SZ 4 +# define BEAM_NATIVE_MIN_FUNC_SZ 4 void erts_release_literal_area(struct ErtsLiteralArea_* literal_area); int erts_is_module_native(BeamCodeHeader* code); diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index b81056c774..d3fd99932a 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -1915,7 +1915,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm return_term, Eterm *refp, erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); erts_dsprintf(dsbufp, "Discarding message %T from %T to %T in an old " - "incarnation (%d) of this node (%d)\n", + "incarnation (%u) of this node (%u)\n", msg, p->common.id, to, @@ -1959,7 +1959,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm return_term, Eterm *refp, erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); erts_dsprintf(dsbufp, "Discarding message %T from %T to %T in an old " - "incarnation (%d) of this node (%d)\n", + "incarnation (%u) of this node (%u)\n", msg, p->common.id, to, @@ -1987,7 +1987,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm return_term, Eterm *refp, trace_send(p, portid, msg); if (have_seqtrace(SEQ_TRACE_TOKEN(p))) { - seq_trace_update_send(p); + seq_trace_update_serial(p); seq_trace_output(SEQ_TRACE_TOKEN(p), msg, SEQ_TRACE_SEND, portid, p); } @@ -4866,9 +4866,13 @@ BIF_RETTYPE phash_2(BIF_ALIST_2) BIF_RETTYPE phash2_1(BIF_ALIST_1) { Uint32 hash; - - hash = make_hash2(BIF_ARG_1); - BIF_RET(make_small(hash & ((1L << 27) - 1))); + Eterm trap_state = THE_NON_VALUE; + hash = trapping_make_hash2(BIF_ARG_1, &trap_state, BIF_P); + if (trap_state == THE_NON_VALUE) { + BIF_RET(make_small(hash & ((1L << 27) - 1))); + } else { + BIF_TRAP1(bif_export[BIF_phash2_1], BIF_P, trap_state); + } } BIF_RETTYPE phash2_2(BIF_ALIST_2) @@ -4876,6 +4880,7 @@ BIF_RETTYPE phash2_2(BIF_ALIST_2) Uint32 hash; Uint32 final_hash; Uint32 range; + Eterm trap_state = THE_NON_VALUE; /* Check for special case 2^32 */ if (term_equals_2pow32(BIF_ARG_2)) { @@ -4887,7 +4892,10 @@ BIF_RETTYPE phash2_2(BIF_ALIST_2) } range = (Uint32) u; } - hash = make_hash2(BIF_ARG_1); + hash = trapping_make_hash2(BIF_ARG_1, &trap_state, BIF_P); + if (trap_state != THE_NON_VALUE) { + BIF_TRAP2(bif_export[BIF_phash2_2], BIF_P, trap_state, BIF_ARG_2); + } if (range) { final_hash = hash % range; /* [0..range-1] */ } else { @@ -4963,15 +4971,28 @@ void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a, Eterm (*bif)(BIF_ALIST)) { int i; + sys_memset((void *) ep, 0, sizeof(Export)); + for (i=0; i<ERTS_NUM_CODE_IX; i++) { - ep->addressv[i] = ep->beam; + ep->addressv[i] = ep->trampoline.raw; } + ep->info.mfa.module = m; ep->info.mfa.function = f; ep->info.mfa.arity = a; - ep->beam[0] = BeamOpCodeAddr(op_apply_bif); - ep->beam[1] = (BeamInstr) bif; + ep->trampoline.op = BeamOpCodeAddr(op_call_bif_W); + ep->trampoline.raw[1] = (BeamInstr)bif; +} + +/* + * Writes a BIF call wrapper to the given address. + */ +void erts_write_bif_wrapper(Export *export, BeamInstr *address) { + BifEntry *entry = &bif_table[export->bif_table_index]; + + address[0] = BeamOpCodeAddr(op_call_bif_W); + address[1] = (BeamInstr)entry->f; } void erts_init_bif(void) @@ -5023,7 +5044,7 @@ void erts_init_bif(void) } /* - * Scheduling of BIFs via NifExport... + * Scheduling of BIFs via ErtsNativeFunc... */ #define ERTS_WANT_NFUNC_SCHED_INTERNALS__ #include "erl_nfunc_sched.h" @@ -5038,8 +5059,8 @@ schedule(Process *c_p, Process *dirty_shadow_proc, int argc, Eterm *argv) { ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p)); - (void) erts_nif_export_schedule(c_p, dirty_shadow_proc, - mfa, pc, BeamOpCodeAddr(op_apply_bif), + (void) erts_nfunc_schedule(c_p, dirty_shadow_proc, + mfa, pc, BeamOpCodeAddr(op_call_bif_W), dfunc, ifunc, module, function, argc, argv); @@ -5048,23 +5069,23 @@ schedule(Process *c_p, Process *dirty_shadow_proc, static BIF_RETTYPE dirty_bif_result(BIF_ALIST_1) { - NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(BIF_P); - erts_nif_export_restore(BIF_P, nep, BIF_ARG_1); + ErtsNativeFunc *nep = (ErtsNativeFunc *) ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(BIF_P); + erts_nfunc_restore(BIF_P, nep, BIF_ARG_1); BIF_RET(BIF_ARG_1); } static BIF_RETTYPE dirty_bif_trap(BIF_ALIST) { - NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(BIF_P); + ErtsNativeFunc *nep = (ErtsNativeFunc *) ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(BIF_P); /* * Arity and argument registers already set * correct by call to dirty_bif_trap()... */ - ASSERT(BIF_P->arity == nep->exp.info.mfa.arity); + ASSERT(BIF_P->arity == nep->trampoline.info.mfa.arity); - erts_nif_export_restore(BIF_P, nep, THE_NON_VALUE); + erts_nfunc_restore(BIF_P, nep, THE_NON_VALUE); BIF_P->i = (BeamInstr *) nep->func; BIF_P->freason = TRAP; @@ -5079,8 +5100,8 @@ static BIF_RETTYPE dirty_bif_exception(BIF_ALIST_2) freason = signed_val(BIF_ARG_1); - /* Restore orig info for error and clear nif export in handle_error() */ - freason |= EXF_RESTORE_NIF; + /* Restore orig info for error and clear nif wrapper in handle_error() */ + freason |= EXF_RESTORE_NFUNC; BIF_P->fvalue = BIF_ARG_2; @@ -5118,6 +5139,7 @@ erts_schedule_bif(Process *proc, if (!ERTS_PROC_IS_EXITING(c_p)) { Export *exp; BifFunction dbif, ibif; + BeamInstr call_instr; BeamInstr *pc; /* @@ -5152,29 +5174,41 @@ erts_schedule_bif(Process *proc, if (i == NULL) { ERTS_INTERNAL_ERROR("Missing instruction pointer"); } + + if (BeamIsOpCode(*i, op_i_generic_breakpoint)) { + ErtsCodeInfo *ci; + GenericBp *bp; + + ci = erts_code_to_codeinfo(i); + bp = ci->u.gen_bp; + + call_instr = bp->orig_instr; + } else { + call_instr = *i; + } + #ifdef HIPE - else if (proc->flags & F_HIPE_MODE) { + if (proc->flags & F_HIPE_MODE) { /* Pointer to bif export in i */ exp = (Export *) i; - pc = c_p->cp; + pc = cp_val(c_p->stop[0]); mfa = &exp->info.mfa; - } + } else /* !! This is part of the if clause below !! */ #endif - else if (BeamIsOpCode(*i, op_call_bif_e)) { - /* Pointer to bif export in i+1 */ - exp = (Export *) i[1]; + if (BeamIsOpCode(call_instr, op_call_light_bif_be)) { + /* Pointer to bif export in i+2 */ + exp = (Export *) i[2]; pc = i; mfa = &exp->info.mfa; } - else if (BeamIsOpCode(*i, op_call_bif_only_e)) { - /* Pointer to bif export in i+1 */ - exp = (Export *) i[1]; + else if (BeamIsOpCode(call_instr, op_call_light_bif_only_be)) { + /* Pointer to bif export in i+2 */ + exp = (Export *) i[2]; pc = i; mfa = &exp->info.mfa; } - else if (BeamIsOpCode(*i, op_apply_bif)) { - /* Pointer to bif in i+1, and mfa in i-3 */ - pc = c_p->cp; + else if (BeamIsOpCode(call_instr, op_call_bif_W)) { + pc = cp_val(c_p->stop[0]); mfa = erts_code_to_codemfa(i); } else { @@ -5202,7 +5236,7 @@ erts_schedule_bif(Process *proc, static BIF_RETTYPE call_bif(Process *c_p, Eterm *reg, BeamInstr *I) { - NifExport *nep = ERTS_I_BEAM_OP_TO_NIF_EXPORT(I); + ErtsNativeFunc *nep = ERTS_I_BEAM_OP_TO_NFUNC(I); ErtsBifFunc bif = (ErtsBifFunc) nep->func; BIF_RETTYPE ret; @@ -5215,12 +5249,12 @@ call_bif(Process *c_p, Eterm *reg, BeamInstr *I) ret = (*bif)(c_p, reg, I); if (is_value(ret)) - erts_nif_export_restore(c_p, nep, ret); + erts_nfunc_restore(c_p, nep, ret); else if (c_p->freason != TRAP) - c_p->freason |= EXF_RESTORE_NIF; /* restore in handle_error() */ + c_p->freason |= EXF_RESTORE_NFUNC; /* restore in handle_error() */ else if (nep->func == ERTS_SCHED_BIF_TRAP_MARKER) { /* BIF did an ordinary trap... */ - erts_nif_export_restore(c_p, nep, ret); + erts_nfunc_restore(c_p, nep, ret); } /* else: * BIF rescheduled itself using erts_schedule_bif(). @@ -5237,7 +5271,7 @@ erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm * int exiting; Process *dirty_shadow_proc; ErtsBifFunc bf; - NifExport *nep; + ErtsNativeFunc *nep; #ifdef DEBUG Eterm *c_p_htop; erts_aint32_t state; @@ -5250,8 +5284,8 @@ erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm * #endif - nep = ERTS_I_BEAM_OP_TO_NIF_EXPORT(I); - ASSERT(nep == ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p)); + nep = ERTS_I_BEAM_OP_TO_NFUNC(I); + ASSERT(nep == ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(c_p)); nep->func = ERTS_SCHED_BIF_TRAP_MARKER; @@ -5265,7 +5299,6 @@ erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm * dirty_shadow_proc->freason = c_p->freason; dirty_shadow_proc->fvalue = c_p->fvalue; dirty_shadow_proc->ftrace = c_p->ftrace; - dirty_shadow_proc->cp = c_p->cp; dirty_shadow_proc->i = c_p->i; #ifdef DEBUG @@ -5312,7 +5345,6 @@ erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm * c_p->freason = dirty_shadow_proc->freason; c_p->fvalue = dirty_shadow_proc->fvalue; c_p->ftrace = dirty_shadow_proc->ftrace; - c_p->cp = dirty_shadow_proc->cp; c_p->i = dirty_shadow_proc->i; c_p->arity = dirty_shadow_proc->arity; } diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index c9f5177bd3..19dabc0514 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -25,13 +25,14 @@ # # <bif-decl> ::= "bif" <bif> <C-name>* | # "ubif" <bif> <C-name>* | -# "gcbif" <bif> <C-name>* +# "hbif" <bif> <C-name>* # <bif> ::= <module> ":" <name> "/" <arity> # -# ubif: Use for operators and guard BIFs that never build anything -# on the heap (such as tuple_size/1) and operators. +# ubif: Use for operators and guard BIFs. # -# gcbif: Use for guard BIFs that may build on the heap (such as abs/1). +# hbif: Use for BIFs that perform garbage collection or need up-to-date +# information on where they were called from. These must be called +# through the export entry. # # bif: Use for all other BIFs. # @@ -60,7 +61,7 @@ bif erlang:display_string/1 bif erlang:display_nl/0 ubif erlang:element/2 bif erlang:erase/0 -bif erlang:erase/1 +hbif erlang:erase/1 bif erlang:exit/1 bif erlang:exit/2 bif erlang:exit_signal/2 @@ -70,7 +71,7 @@ ubif erlang:float/1 bif erlang:float_to_list/1 bif erlang:float_to_list/2 bif erlang:fun_info/2 -bif erts_internal:garbage_collect/1 +hbif erts_internal:garbage_collect/1 bif erlang:get/0 bif erlang:get/1 bif erlang:get_keys/1 @@ -127,10 +128,10 @@ bif erlang:ports/0 bif erlang:pre_loaded/0 bif erlang:process_flag/2 bif erts_internal:process_flag/3 -bif erlang:process_info/1 -bif erlang:process_info/2 +hbif erlang:process_info/1 +hbif erlang:process_info/2 bif erlang:processes/0 -bif erlang:put/2 +hbif erlang:put/2 bif erlang:register/2 bif erlang:registered/0 ubif erlang:round/1 @@ -174,7 +175,7 @@ bif erts_internal:port_connect/2 bif erts_internal:request_system_task/3 bif erts_internal:request_system_task/4 -bif erts_internal:check_process_code/1 +hbif erts_internal:check_process_code/1 bif erts_internal:map_to_tuple_keys/1 bif erts_internal:term_type/1 @@ -466,7 +467,7 @@ bif code:is_module_native/1 # New Bifs in R9C. # -bif erlang:hibernate/3 +hbif erlang:hibernate/3 bif error_logger:warning_map/0 # diff --git a/erts/emulator/beam/bif_instrs.tab b/erts/emulator/beam/bif_instrs.tab index 8e0caa38a3..de5305bde4 100644 --- a/erts/emulator/beam/bif_instrs.tab +++ b/erts/emulator/beam/bif_instrs.tab @@ -212,26 +212,32 @@ i_length.execute(Fail, Live, Dst) { // Call a BIF, store the result in x(0) and transfer control to the // next instruction. // -call_bif(Exp) { +call_light_bif(Bif, Exp) { + Export *export; ErtsBifFunc bf; + Eterm result; ErlHeapFragment *live_hf_end; - Export *export = (Export*) $Exp; + + bf = (ErtsBifFunc) $Bif; + export = (Export*) $Exp; if (!((FCALLS - 1) > 0 || (FCALLS-1) > neg_o_reds)) { /* * If we have run out of reductions, do a context * switch before calling the BIF. */ - c_p->arity = GET_BIF_ARITY(export); + c_p->arity = GET_EXPORT_ARITY(export); c_p->current = &export->info.mfa; goto context_switch3; } - ERTS_MSACC_SET_BIF_STATE_CACHED_X(GET_BIF_MODULE(export), - GET_BIF_ADDRESS(export)); + if (ERTS_UNLIKELY(export->is_bif_traced)) { + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); + $DISPATCH_EXPORT(export); + } - bf = GET_BIF_ADDRESS(export); + ERTS_MSACC_SET_BIF_STATE_CACHED_X(GET_EXPORT_MODULE(export), bf); PRE_BIF_SWAPOUT(c_p); ERTS_DBG_CHK_REDS(c_p, FCALLS); @@ -243,21 +249,26 @@ call_bif(Exp) { ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); live_hf_end = c_p->mbuf; ERTS_CHK_MBUF_SZ(c_p); + result = (*bf)(c_p, reg, I); + + /* Only heavy BIFs may GC. */ + ASSERT(E == c_p->stop); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_HOLE_CHECK(c_p); ERTS_REQ_PROC_MAIN_LOCK(c_p); if (ERTS_IS_GC_DESIRED(c_p)) { - Uint arity = GET_BIF_ARITY(export); + Uint arity = GET_EXPORT_ARITY(export); result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, result, reg, arity); E = c_p->stop; } - PROCESS_MAIN_CHK_LOCKS(c_p); HTOP = HEAP_TOP(c_p); FCALLS = c_p->fcalls; + PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_DBG_CHK_REDS(c_p, FCALLS); /* @@ -280,10 +291,9 @@ call_bif(Exp) { * erlang code or by nif_bif.epilogue() when the BIF * is done). */ - SET_CP(c_p, $NEXT_INSTRUCTION); + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); SET_I(c_p->i); - SWAPIN; - Dispatch(); + $DISPATCH(); } /* @@ -297,29 +307,38 @@ call_bif(Exp) { // // Call a BIF tail-recursively, storing the result in x(0) and doing -// a return to the continuation poiner (c_p->cp). +// a return to the continuation poiner. // - -call_bif_only(Exp) { +call_light_bif_only(Bif, Exp) { + ErlHeapFragment *live_hf_end; ErtsBifFunc bf; + Export *export; Eterm result; - ErlHeapFragment *live_hf_end; - Export *export = (Export*) $Exp; + + bf = (ErtsBifFunc) $Bif; + export = (Export*) $Exp; if (!((FCALLS - 1) > 0 || (FCALLS-1) > neg_o_reds)) { /* * If we have run out of reductions, do a context * switch before calling the BIF. */ - c_p->arity = GET_BIF_ARITY(export); + c_p->arity = GET_EXPORT_ARITY(export); c_p->current = &export->info.mfa; goto context_switch3; } - ERTS_MSACC_SET_BIF_STATE_CACHED_X(GET_BIF_MODULE(export), - GET_BIF_ADDRESS(export)); + if (ERTS_UNLIKELY(export->is_bif_traced)) { + /* Set up a dummy stack frame so we can perform a normal call. Loader + * transformations ensure that the next instruction after this is + * 'deallocate_return 0'. */ + $AH(0, 0, GET_EXPORT_ARITY(export)); + + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); + $DISPATCH_EXPORT(export); + } - bf = GET_BIF_ADDRESS(export); + ERTS_MSACC_SET_BIF_STATE_CACHED_X(GET_EXPORT_MODULE(export), bf); PRE_BIF_SWAPOUT(c_p); ERTS_DBG_CHK_REDS(c_p, FCALLS); @@ -331,21 +350,26 @@ call_bif_only(Exp) { ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); live_hf_end = c_p->mbuf; ERTS_CHK_MBUF_SZ(c_p); + result = (*bf)(c_p, reg, I); + + /* Only heavy BIFs may GC. */ + ASSERT(E == c_p->stop); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_HOLE_CHECK(c_p); ERTS_REQ_PROC_MAIN_LOCK(c_p); if (ERTS_IS_GC_DESIRED(c_p)) { - Uint arity = GET_BIF_ARITY(export); + Uint arity = GET_EXPORT_ARITY(export); result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, result, reg, arity); E = c_p->stop; } - PROCESS_MAIN_CHK_LOCKS(c_p); HTOP = HEAP_TOP(c_p); FCALLS = c_p->fcalls; + PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_DBG_CHK_REDS(c_p, FCALLS); /* @@ -367,11 +391,10 @@ call_bif_only(Exp) { } else if (c_p->freason == TRAP) { /* * Dispatch to a trap. When the trap is done, a jump - * to the continuation pointer (c_p->cp) will be done. + * to the continuation pointer on the stack will be done. */ SET_I(c_p->i); - SWAPIN; - Dispatch(); + $DISPATCH(); } /* @@ -413,17 +436,17 @@ send() { r(0) = result; CHECK_TERM(r(0)); } else if (c_p->freason == TRAP) { - SET_CP(c_p, $NEXT_INSTRUCTION); + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); SET_I(c_p->i); SWAPIN; - Dispatch(); + $DISPATCH(); } else { goto find_func_info; } } call_nif := nif_bif.call_nif.epilogue; -apply_bif := nif_bif.apply_bif.epilogue; +call_bif := nif_bif.call_bif.epilogue; nif_bif.head() { Eterm nif_bif_result; @@ -433,7 +456,7 @@ nif_bif.head() { ErtsCodeMFA *codemfa; } -nif_bif.call_nif() { +nif_bif.call_nif(Func, NifMod, DirtyFunc) { /* * call_nif is always first instruction in function: * @@ -443,11 +466,14 @@ nif_bif.call_nif() { * I[0]: &&call_nif * I[1]: Function pointer to NIF function * I[2]: Pointer to erl_module_nif - * I[3]: Function pointer to dirty NIF + * I[3]: Function pointer to dirty NIF. This is not used in this + * instruction, but dirty schedulers look at it. * - * This layout is determined by the NifExport struct + * This layout is determined by the ErtsNativeFunc struct */ + (void)$DirtyFunc; + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); codemfa = erts_code_to_codemfa(I); @@ -465,12 +491,12 @@ nif_bif.call_nif() { ASSERT(!ERTS_PROC_IS_EXITING(c_p)); { typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]); - NifF* fp = vbf = (NifF*) I[1]; + NifF* fp = vbf = (NifF*) $Func; struct enif_environment_t env; ASSERT(c_p->scheduler_data); live_hf_end = c_p->mbuf; ERTS_CHK_MBUF_SZ(c_p); - erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL); + erts_pre_nif(&env, c_p, (struct erl_module_nif*)$NifMod, NULL); ASSERT((c_p->scheduler_data)->current_nif == NULL); (c_p->scheduler_data)->current_nif = &env; @@ -495,15 +521,15 @@ nif_bif.call_nif() { DTRACE_NIF_RETURN(c_p, codemfa); } -nif_bif.apply_bif() { +nif_bif.call_bif(Func) { /* - * At this point, I points to the code[0] in the export entry for - * the BIF: + * At this point, I points to the code[0] in the native function wrapper + * for the BIF: * * code[-3]: Module * code[-2]: Function * code[-1]: Arity - * code[0]: &&apply_bif + * code[0]: &&call_bif * code[1]: Function pointer to BIF function */ @@ -515,21 +541,19 @@ nif_bif.apply_bif() { codemfa = erts_code_to_codemfa(I); - ERTS_MSACC_SET_BIF_STATE_CACHED_X(codemfa->module, (BifFunction)Arg(0)); - + ERTS_MSACC_SET_BIF_STATE_CACHED_X(codemfa->module, (BifFunction)$Func); /* In case we apply process_info/1,2 or load_nif/1 */ c_p->current = codemfa; $SET_CP_I_ABS(I); /* In case we apply check_process_code/2. */ c_p->arity = 0; /* To allow garbage collection on ourselves - * (check_process_code/2). - */ + * (check_process_code/2, put/2, etc). */ DTRACE_BIF_ENTRY(c_p, codemfa); SWAPOUT; ERTS_DBG_CHK_REDS(c_p, FCALLS - 1); c_p->fcalls = FCALLS - 1; - vbf = (BifFunction) Arg(0); + vbf = (BifFunction)$Func; PROCESS_MAIN_CHK_LOCKS(c_p); bif_nif_arity = codemfa->arity; ASSERT(bif_nif_arity <= 4); @@ -557,6 +581,7 @@ nif_bif.apply_bif() { } nif_bif.epilogue() { + //| -no_next ERTS_REQ_PROC_MAIN_LOCK(c_p); ERTS_HOLE_CHECK(c_p); if (ERTS_IS_GC_DESIRED(c_p)) { @@ -570,8 +595,7 @@ nif_bif.epilogue() { if (ERTS_LIKELY(is_value(nif_bif_result))) { r(0) = nif_bif_result; CHECK_TERM(r(0)); - SET_I(c_p->cp); - c_p->cp = 0; + $RETURN(); Goto(*I); } else if (c_p->freason == TRAP) { SET_I(c_p->i); @@ -579,8 +603,42 @@ nif_bif.epilogue() { c_p->flags &= ~F_HIBERNATE_SCHED; goto do_schedule; } - Dispatch(); + $DISPATCH(); + } + { + BeamInstr *cp = erts_printable_return_address(c_p, E); + ASSERT(VALID_INSTR(*cp)); + I = handle_error(c_p, cp, reg, c_p->current); } - I = handle_error(c_p, c_p->cp, reg, c_p->current); goto post_error_handling; } + +i_load_nif() { + //| -no_next + if (erts_try_seize_code_write_permission(c_p)) { + Eterm result; + + PRE_BIF_SWAPOUT(c_p); + result = erts_load_nif(c_p, I, r(0), r(1)); + erts_release_code_write_permission(); + ERTS_REQ_PROC_MAIN_LOCK(c_p); + SWAPIN; + + if (ERTS_LIKELY(is_value(result))) { + r(0) = result; + $NEXT0(); + } else { + static ErtsCodeMFA mfa = {am_erlang, am_load_nif, 2}; + c_p->freason = BADARG; + I = handle_error(c_p, I, reg, &mfa); + goto post_error_handling; + } + } else { + /* Yield and try again */ + $SET_CP_I_ABS(I); + SWAPOUT; + c_p->current = NULL; + c_p->arity = 2; + goto do_schedule; + } +} diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c index 522f50287a..7666f23a4f 100644 --- a/erts/emulator/beam/big.c +++ b/erts/emulator/beam/big.c @@ -2176,6 +2176,24 @@ term_to_Uint64(Eterm term, Uint64 *up) #endif } +int +term_to_Uint32(Eterm term, Uint32 *up) +{ +#if ERTS_SIZEOF_ETERM == 4 + return term_to_Uint(term,up); +#else + if (is_small(term)) { + Sint i = signed_val(term); + if (i >= 0) { + *up = (Uint32) i; + return 1; + } + } + *up = BADARG; + return 0; +#endif +} + int term_to_Sint(Eterm term, Sint *sp) { diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index ad19cce395..3fed076419 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -168,6 +168,8 @@ Eterm erts_uint64_array_to_big(Uint **, int, int, Uint64 *); int term_to_Uint64(Eterm, Uint64*); int term_to_Sint64(Eterm, Sint64*); #endif +int term_to_Uint32(Eterm, Uint32*); + Uint32 big_to_uint32(Eterm b); int term_equals_2pow32(Eterm); diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index eb9e749a08..dafe805a6f 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -1051,7 +1051,7 @@ erts_dsig_send_msg(ErtsDSigSendContext* ctx, Eterm remote, Eterm message) #endif if (have_seqtrace(SEQ_TRACE_TOKEN(sender))) { - seq_trace_update_send(sender); + seq_trace_update_serial(sender); token = SEQ_TRACE_TOKEN(sender); seq_trace_output(token, message, SEQ_TRACE_SEND, remote, sender); } @@ -1125,7 +1125,7 @@ erts_dsig_send_reg_msg(ErtsDSigSendContext* ctx, Eterm remote_name, Eterm messag #endif if (have_seqtrace(SEQ_TRACE_TOKEN(sender))) { - seq_trace_update_send(sender); + seq_trace_update_serial(sender); token = SEQ_TRACE_TOKEN(sender); seq_trace_output(token, message, SEQ_TRACE_SEND, remote_name, sender); } @@ -1184,7 +1184,7 @@ erts_dsig_send_exit_tt(ErtsDSigSendContext *ctx, Eterm local, Eterm remote, msg = reason; if (have_seqtrace(token)) { - seq_trace_update_send(ctx->c_p); + seq_trace_update_serial(ctx->c_p); seq_trace_output_exit(token, reason, SEQ_TRACE_SEND, remote, local); if (ctx->dep->flags & DFLAG_EXIT_PAYLOAD) { ctl = TUPLE4(&ctx->ctl_heap[0], @@ -3762,12 +3762,10 @@ int distribution_info(fmtfn_t to, void *arg) /* Called by break handler */ BIF_RETTYPE setnode_2(BIF_ALIST_2) { Process *net_kernel; - Uint creation; + Uint32 creation; /* valid creation ? */ - if(!term_to_Uint(BIF_ARG_2, &creation)) - goto error; - if(creation > 3) + if(!term_to_Uint32(BIF_ARG_2, &creation)) goto error; /* valid node name ? */ @@ -3811,7 +3809,7 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2) erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_thr_progress_block(); inc_no_nodes(); - erts_set_this_node(BIF_ARG_1, (Uint32) creation); + erts_set_this_node(BIF_ARG_1, creation); erts_is_alive = 1; send_nodes_mon_msgs(NULL, am_nodeup, BIF_ARG_1, am_visible, NIL); erts_thr_progress_unblock(); diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index a33fb7efcf..37ec88cc55 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -54,11 +54,12 @@ #define DFLAG_DIST_MANDATORY (DFLAG_EXTENDED_REFERENCES \ | DFLAG_EXTENDED_PIDS_PORTS \ | DFLAG_UTF8_ATOMS \ - | DFLAG_NEW_FUN_TAGS) + | DFLAG_NEW_FUN_TAGS \ + | DFLAG_BIG_CREATION) /* * Additional optimistic flags when encoding toward pending connection. - * If remote node (erl_interface) does not supporting these then we may need + * If remote node (erl_interface) does not support these then we may need * to transcode messages enqueued before connection setup was finished. */ #define DFLAG_DIST_HOPEFULLY (DFLAG_EXPORT_PTR_TAG \ @@ -75,7 +76,6 @@ | DFLAG_SMALL_ATOM_TAGS \ | DFLAG_UTF8_ATOMS \ | DFLAG_MAP_TAG \ - | DFLAG_BIG_CREATION \ | DFLAG_SEND_SENDER \ | DFLAG_BIG_SEQTRACE_LABELS \ | DFLAG_EXIT_PAYLOAD \ diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index b9f0334172..1bbc7d7f1e 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -653,8 +653,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) = erts_timer_type_size(ERTS_ALC_T_HL_PTIMER); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_BIF_TIMER)] = erts_timer_type_size(ERTS_ALC_T_BIF_TIMER); - fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NIF_EXP_TRACE)] - = sizeof(NifExportTrace); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MREF_NSCHED_ENT)] = sizeof(ErtsNSchedMagicRefTableEntry); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MINDIRECTION)] @@ -2392,10 +2390,6 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg) &size.processes_used, fi, ERTS_ALC_T_BIF_TIMER); - add_fix_values(&size.processes, - &size.processes_used, - fi, - ERTS_ALC_T_NIF_EXP_TRACE); } if (want.atom || want.atom_used) { diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 21941ba96e..3e643a6223 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -278,6 +278,7 @@ type SETUP_CONN_ARG SHORT_LIVED PROCESSES setup_connection_argument type LIST_TRAP SHORT_LIVED PROCESSES list_bif_trap_state type CONT_EXIT_TRAP SHORT_LIVED PROCESSES continue_exit_trap_state type SEQ_YIELD_STATE SHORT_LIVED SYSTEM dist_seq_yield_state +type PHASH2_TRAP SHORT_LIVED PROCESSES phash2_trap_state type ENVIRONMENT SYSTEM SYSTEM environment @@ -330,8 +331,7 @@ type DB_HEIR_DATA STANDARD ETS db_heir_data type DB_MS_PSDO_PROC LONG_LIVED ETS db_match_pseudo_proc type SCHDLR_DATA LONG_LIVED SYSTEM scheduler_data -type NIF_TRAP_EXPORT STANDARD PROCESSES nif_trap_export_entry -type NIF_EXP_TRACE FIXED_SIZE PROCESSES nif_export_trace +type NFUNC_TRAP_WRAPPER STANDARD PROCESSES nfunc_trap_wrapper type EXPORT LONG_LIVED CODE export_entry type MONITOR FIXED_SIZE PROCESSES monitor type MONITOR_SUSPEND STANDARD PROCESSES monitor_suspend diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index b06b5fc1ab..57fc5ec131 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -158,8 +158,9 @@ static Eterm os_version_tuple; static Eterm current_function(Process* p, ErtsHeapFactory *hfact, Process* rp, int full_info, Uint reserve_size, int flags); -static Eterm current_stacktrace(ErtsHeapFactory *hfact, Process* rp, - Uint reserve_size); +static Eterm +current_stacktrace(Process* p, ErtsHeapFactory *hfact, Process* rp, + Uint reserve_size, int flags); Eterm erts_bld_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh, Eterm tail) @@ -1384,7 +1385,7 @@ process_info_aux(Process *c_p, break; case ERTS_PI_IX_CURRENT_STACKTRACE: - res = current_stacktrace(hfact, rp, reserve_size); + res = current_stacktrace(c_p, hfact, rp, reserve_size, flags); break; case ERTS_PI_IX_INITIAL_CALL: @@ -2022,19 +2023,23 @@ current_function(Process *c_p, ErtsHeapFactory *hfact, Process* rp, } if (c_p == rp && !(flags & ERTS_PI_FLAG_REQUEST_FOR_OTHER)) { - FunctionInfo fi2; + BeamInstr* return_address; + FunctionInfo caller_fi; - /* - * The current function is erlang:process_info/{1,2}, - * which is not the answer that the application want. - * We will use the function pointed into by rp->cp - * instead if it can be looked up. - */ - erts_lookup_function_info(&fi2, rp->cp, full_info); - if (fi2.mfa) { - fi = fi2; - rp->current = fi2.mfa; - } + /* + * The current function is erlang:process_info/{1,2}, and we've + * historically returned the *calling* function in that case. We + * therefore use the continuation pointer stored at the top of the + * stack instead, which is safe since process_info is a "heavy" BIF + * that is only called through its export entry. + */ + return_address = erts_printable_return_address(rp, STACK_TOP(rp)); + + erts_lookup_function_info(&caller_fi, return_address, full_info); + if (caller_fi.mfa) { + fi = caller_fi; + rp->current = caller_fi.mfa; + } } /* @@ -2055,8 +2060,8 @@ current_function(Process *c_p, ErtsHeapFactory *hfact, Process* rp, } static Eterm -current_stacktrace(ErtsHeapFactory *hfact, Process* rp, - Uint reserve_size) +current_stacktrace(Process *p, ErtsHeapFactory *hfact, Process* rp, + Uint reserve_size, int flags) { Uint sz; struct StackTrace* s; @@ -2073,13 +2078,14 @@ current_stacktrace(ErtsHeapFactory *hfact, Process* rp, sz = offsetof(struct StackTrace, trace) + sizeof(BeamInstr *)*depth; s = (struct StackTrace *) erts_alloc(ERTS_ALC_T_TMP, sz); s->depth = 0; - if (depth > 0 && rp->i) { - s->trace[s->depth++] = rp->i; - depth--; - } - if (depth > 0 && rp->cp != 0) { - s->trace[s->depth++] = rp->cp - 1; - depth--; + s->pc = NULL; + + /* We skip current pc when requesting our own stack trace since it will + * inevitably point to process_info/1,2 */ + if ((p != rp || (flags & ERTS_PI_FLAG_REQUEST_FOR_OTHER)) && + depth > 0 && rp->i) { + s->trace[s->depth++] = rp->i; + depth--; } erts_save_stacktrace(rp, s, depth); @@ -2814,7 +2820,10 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) } else if (BIF_ARG_1 == am_threads) { return am_true; } else if (BIF_ARG_1 == am_creation) { - return make_small(erts_this_node->creation); + Uint hsz = 0; + erts_bld_uint(NULL, &hsz, erts_this_node->creation); + hp = hsz ? HAlloc(BIF_P, hsz) : NULL; + BIF_RET(erts_bld_uint(&hp, NULL, erts_this_node->creation)); } else if (BIF_ARG_1 == am_break_ignored) { extern int ignore_break; if (ignore_break) diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c index fa2edfef1e..9d485abc35 100644 --- a/erts/emulator/beam/erl_bif_lists.c +++ b/erts/emulator/beam/erl_bif_lists.c @@ -32,8 +32,46 @@ #include "bif.h" #include "erl_binary.h" +static Export plusplus_trap_export; +static Export append_trap_export; -static Eterm keyfind(int Bif, Process* p, Eterm Key, Eterm Pos, Eterm List); +static Export minusminus_trap_export; +static Export subtract_trap_export; + +static Export member_trap_export; + +static Export reverse_trap_export; + +static Export keymember_trap_export; +static Export keysearch_trap_export; +static Export keyfind_trap_export; + +void erts_init_bif_lists(void) { + erts_init_trap_export(&plusplus_trap_export, am_erlang, am_PlusPlus, 2, + ebif_plusplus_2); + erts_init_trap_export(&append_trap_export, am_erlang, am_append, 2, + append_2); + + erts_init_trap_export(&minusminus_trap_export, am_erlang, am_MinusMinus, 2, + ebif_minusminus_2); + erts_init_trap_export(&subtract_trap_export, am_lists, am_subtract, 2, + subtract_2); + + erts_init_trap_export(&reverse_trap_export, am_lists, am_reverse, 2, + lists_reverse_2); + + erts_init_trap_export(&member_trap_export, am_lists, am_member, 2, + lists_member_2); + + erts_init_trap_export(&keymember_trap_export, am_lists, am_keymember, 3, + lists_keymember_3); + erts_init_trap_export(&keysearch_trap_export, am_lists, am_keysearch, 3, + lists_keysearch_3); + erts_init_trap_export(&keyfind_trap_export, am_lists, am_keyfind, 3, + lists_keyfind_3); +} + +static Eterm keyfind(Export* Bif, Process* p, Eterm Key, Eterm Pos, Eterm List); /* erlang:'++'/2 * @@ -308,12 +346,12 @@ static Eterm append(Export *bif_entry, BIF_ALIST_2) { Eterm ebif_plusplus_2(BIF_ALIST_2) { - return append(bif_export[BIF_ebif_plusplus_2], BIF_CALL_ARGS); + return append(&plusplus_trap_export, BIF_CALL_ARGS); } BIF_RETTYPE append_2(BIF_ALIST_2) { - return append(bif_export[BIF_append_2], BIF_CALL_ARGS); + return append(&append_trap_export, BIF_CALL_ARGS); } /* erlang:'--'/2 @@ -1039,11 +1077,11 @@ static Eterm subtract(Export *bif_entry, BIF_ALIST_2) { } BIF_RETTYPE ebif_minusminus_2(BIF_ALIST_2) { - return subtract(bif_export[BIF_ebif_minusminus_2], BIF_CALL_ARGS); + return subtract(&minusminus_trap_export, BIF_CALL_ARGS); } BIF_RETTYPE subtract_2(BIF_ALIST_2) { - return subtract(bif_export[BIF_subtract_2], BIF_CALL_ARGS); + return subtract(&subtract_trap_export, BIF_CALL_ARGS); } @@ -1068,7 +1106,7 @@ BIF_RETTYPE lists_member_2(BIF_ALIST_2) while (is_list(list)) { if (--max_iter < 0) { BUMP_ALL_REDS(BIF_P); - BIF_TRAP2(bif_export[BIF_lists_member_2], BIF_P, term, list); + BIF_TRAP2(&member_trap_export, BIF_P, term, list); } item = CAR(list_val(list)); if ((item == term) || (non_immed_key && eq(item, term))) { @@ -1130,7 +1168,7 @@ static BIF_RETTYPE lists_reverse_alloc(Process *c_p, } ASSERT(is_list(tail) && cells_left == 0); - BIF_TRAP2(bif_export[BIF_lists_reverse_2], c_p, list, tail); + BIF_TRAP2(&reverse_trap_export, c_p, list, tail); } static BIF_RETTYPE lists_reverse_onheap(Process *c_p, @@ -1179,7 +1217,7 @@ static BIF_RETTYPE lists_reverse_onheap(Process *c_p, } BUMP_ALL_REDS(c_p); - BIF_TRAP2(bif_export[BIF_lists_reverse_2], c_p, list, tail); + BIF_TRAP2(&reverse_trap_export, c_p, list, tail); } BIF_ERROR(c_p, BADARG); @@ -1209,7 +1247,7 @@ lists_keymember_3(BIF_ALIST_3) { Eterm res; - res = keyfind(BIF_lists_keymember_3, BIF_P, + res = keyfind(&keymember_trap_export, BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); if (is_value(res) && is_tuple(res)) { return am_true; @@ -1223,7 +1261,7 @@ lists_keysearch_3(BIF_ALIST_3) { Eterm res; - res = keyfind(BIF_lists_keysearch_3, BIF_P, + res = keyfind(&keysearch_trap_export, BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); if (is_non_value(res) || is_not_tuple(res)) { return res; @@ -1236,12 +1274,12 @@ lists_keysearch_3(BIF_ALIST_3) BIF_RETTYPE lists_keyfind_3(BIF_ALIST_3) { - return keyfind(BIF_lists_keyfind_3, BIF_P, + return keyfind(&keyfind_trap_export, BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); } static Eterm -keyfind(int Bif, Process* p, Eterm Key, Eterm Pos, Eterm List) +keyfind(Export *Bif, Process* p, Eterm Key, Eterm Pos, Eterm List) { int max_iter = 10 * CONTEXT_REDS; Sint pos; @@ -1257,7 +1295,7 @@ keyfind(int Bif, Process* p, Eterm Key, Eterm Pos, Eterm List) while (is_list(List)) { if (--max_iter < 0) { BUMP_ALL_REDS(p); - BIF_TRAP3(bif_export[Bif], p, Key, Pos, List); + BIF_TRAP3(Bif, p, Key, Pos, List); } term = CAR(list_val(List)); List = CDR(list_val(List)); @@ -1282,7 +1320,7 @@ keyfind(int Bif, Process* p, Eterm Key, Eterm Pos, Eterm List) while (is_list(List)) { if (--max_iter < 0) { BUMP_ALL_REDS(p); - BIF_TRAP3(bif_export[Bif], p, Key, Pos, List); + BIF_TRAP3(Bif, p, Key, Pos, List); } term = CAR(list_val(List)); List = CDR(list_val(List)); @@ -1300,7 +1338,7 @@ keyfind(int Bif, Process* p, Eterm Key, Eterm Pos, Eterm List) while (is_list(List)) { if (--max_iter < 0) { BUMP_ALL_REDS(p); - BIF_TRAP3(bif_export[Bif], p, Key, Pos, List); + BIF_TRAP3(Bif, p, Key, Pos, List); } term = CAR(list_val(List)); List = CDR(list_val(List)); diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index ed825d3dda..dd1e884705 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -44,6 +44,7 @@ #include "erl_bif_unique.h" #include "dtrace-wrapper.h" #include "erl_proc_sig_queue.h" +#include "erl_osenv.h" static Port *open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump); static int merge_global_environment(erts_osenv_t *env, Eterm key_value_pairs); diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index b31d5b86cb..e03c97fe10 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -80,9 +80,6 @@ static Eterm trace_info_func(Process* p, Eterm pid_spec, Eterm key); static Eterm trace_info_on_load(Process* p, Eterm key); static Eterm trace_info_event(Process* p, Eterm event, Eterm key); - -static void reset_bif_trace(void); -static void setup_bif_trace(void); static void install_exp_breakpoints(BpFunctions* f); static void uninstall_exp_breakpoints(BpFunctions* f); static void clean_export_entries(BpFunctions* f); @@ -1047,14 +1044,13 @@ static int function_is_traced(Process *p, e.info.mfa.function = mfa[1]; e.info.mfa.arity = mfa[2]; if ((ep = export_get(&e)) != NULL) { - pc = ep->beam; + pc = ep->trampoline.raw; if (ep->addressv[erts_active_code_ix()] == pc && ! BeamIsOpCode(*pc, op_call_error_handler)) { int r = 0; - ASSERT(BeamIsOpCode(*pc, op_apply_bif) || - BeamIsOpCode(*pc, op_i_generic_breakpoint)); + ASSERT(BeamIsOpCode(*pc, op_i_generic_breakpoint)); if (erts_is_trace_break(&ep->info, ms, 0)) { return FUNC_TRACE_GLOBAL_TRACE; @@ -1426,18 +1422,21 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified, int n; BpFunction* fp; - /* - * First work on normal functions (not real BIFs). - */ - erts_bp_match_export(&finish_bp.e, mfa, specified); fp = finish_bp.e.matching; n = finish_bp.e.matched; for (i = 0; i < n; i++) { ErtsCodeInfo *ci = fp[i].ci; - BeamInstr* pc = erts_codeinfo_to_code(ci); - Export* ep = ErtsContainerStruct(ci, Export, info); + BeamInstr* pc; + Export* ep; + + pc = erts_codeinfo_to_code(ci); + ep = ErtsContainerStruct(ci, Export, info); + + if (ep->bif_table_index != -1) { + ep->is_bif_traced = !!on; + } if (on && !flags.breakpoint) { /* Turn on global call tracing */ @@ -1446,12 +1445,12 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified, #ifdef DEBUG ep->info.op = BeamOpCodeAddr(op_i_func_info_IaaI); #endif - ep->beam[0] = BeamOpCodeAddr(op_trace_jump_W); - ep->beam[1] = (BeamInstr) ep->addressv[code_ix]; + ep->trampoline.op = BeamOpCodeAddr(op_trace_jump_W); + ep->trampoline.trace.address = (BeamInstr) ep->addressv[code_ix]; } - erts_set_call_trace_bif(ci, match_prog_set, 0); + erts_set_export_trace(ci, match_prog_set, 0); if (ep->addressv[code_ix] != pc) { - ep->beam[0] = BeamOpCodeAddr(op_i_generic_breakpoint); + ep->trampoline.op = BeamOpCodeAddr(op_i_generic_breakpoint); } } else if (!on && flags.breakpoint) { /* Turn off breakpoint tracing -- nothing to do here. */ @@ -1460,91 +1459,14 @@ erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified, * Turn off global tracing, either explicitly or implicitly * before turning on breakpoint tracing. */ - erts_clear_call_trace_bif(ci, 0); - if (BeamIsOpCode(ep->beam[0], op_i_generic_breakpoint)) { - ep->beam[0] = BeamOpCodeAddr(op_trace_jump_W); + erts_clear_export_trace(ci, 0); + if (BeamIsOpCode(ep->trampoline.op, op_i_generic_breakpoint)) { + ep->trampoline.op = BeamOpCodeAddr(op_trace_jump_W); } } } /* - ** OK, now for the bif's - */ - for (i = 0; i < BIF_SIZE; ++i) { - Export *ep = bif_export[i]; - - if (!ExportIsBuiltIn(ep)) { - continue; - } - - if (bif_table[i].f == bif_table[i].traced) { - /* Trace wrapper same as regular function - untraceable */ - continue; - } - - switch (specified) { - case 3: - if (mfa->arity != ep->info.mfa.arity) - continue; - case 2: - if (mfa->function != ep->info.mfa.function) - continue; - case 1: - if (mfa->module != ep->info.mfa.module) - continue; - case 0: - break; - default: - ASSERT(0); - } - - if (! flags.breakpoint) { /* Export entry call trace */ - if (on) { - erts_clear_call_trace_bif(&ep->info, 1); - erts_clear_mtrace_bif(&ep->info); - erts_set_call_trace_bif(&ep->info, match_prog_set, 0); - } else { /* off */ - erts_clear_call_trace_bif(&ep->info, 0); - } - matches++; - } else { /* Breakpoint call trace */ - int m = 0; - - if (on) { - if (flags.local) { - erts_clear_call_trace_bif(&ep->info, 0); - erts_set_call_trace_bif(&ep->info, match_prog_set, 1); - m = 1; - } - if (flags.meta) { - erts_set_mtrace_bif(&ep->info, meta_match_prog_set, - meta_tracer); - m = 1; - } - if (flags.call_time) { - erts_set_time_trace_bif(&ep->info, on); - /* I don't want to remove any other tracers */ - m = 1; - } - } else { /* off */ - if (flags.local) { - erts_clear_call_trace_bif(&ep->info, 1); - m = 1; - } - if (flags.meta) { - erts_clear_mtrace_bif(&ep->info); - m = 1; - } - if (flags.call_time) { - erts_clear_time_trace_bif(&ep->info); - m = 1; - } - } - matches += m; - } - } - - /* ** So, now for breakpoint tracing */ erts_bp_match_functions(&finish_bp.f, mfa, specified); @@ -1670,7 +1592,6 @@ erts_finish_breakpointing(void) install_exp_breakpoints(&finish_bp.e); } } - setup_bif_trace(); return 1; case 1: /* @@ -1699,7 +1620,6 @@ erts_finish_breakpointing(void) uninstall_exp_breakpoints(&finish_bp.e); } } - reset_bif_trace(); return 1; case 3: /* @@ -1710,7 +1630,6 @@ erts_finish_breakpointing(void) * updated). If any breakpoints have been totally disabled, * deallocate the GenericBp structs for them. */ - erts_consolidate_bif_bp_data(); clean_export_entries(&finish_bp.e); erts_consolidate_bp_data(&finish_bp.e, 0); erts_consolidate_bp_data(&finish_bp.f, 1); @@ -1736,7 +1655,7 @@ install_exp_breakpoints(BpFunctions* f) for (i = 0; i < ne; i++) { Export* ep = ErtsContainerStruct(fp[i].ci, Export, info); - ep->addressv[code_ix] = ep->beam; + ep->addressv[code_ix] = ep->trampoline.raw; } } @@ -1751,11 +1670,12 @@ uninstall_exp_breakpoints(BpFunctions* f) for (i = 0; i < ne; i++) { Export* ep = ErtsContainerStruct(fp[i].ci, Export, info); - if (ep->addressv[code_ix] != ep->beam) { - continue; - } - ASSERT(BeamIsOpCode(ep->beam[0], op_trace_jump_W)); - ep->addressv[code_ix] = (BeamInstr *) ep->beam[1]; + if (ep->addressv[code_ix] != ep->trampoline.raw) { + continue; + } + + ASSERT(BeamIsOpCode(ep->trampoline.op, op_trace_jump_W)); + ep->addressv[code_ix] = (BeamInstr *) ep->trampoline.trace.address; } } @@ -1770,48 +1690,14 @@ clean_export_entries(BpFunctions* f) for (i = 0; i < ne; i++) { Export* ep = ErtsContainerStruct(fp[i].ci, Export, info); - if (ep->addressv[code_ix] == ep->beam) { - continue; - } - if (BeamIsOpCode(ep->beam[0], op_trace_jump_W)) { - ep->beam[0] = (BeamInstr) 0; - ep->beam[1] = (BeamInstr) 0; - } - } -} - -static void -setup_bif_trace(void) -{ - int i; - - for (i = 0; i < BIF_SIZE; ++i) { - Export *ep = bif_export[i]; - GenericBp* g = ep->info.u.gen_bp; - if (g) { - if (ExportIsBuiltIn(ep)) { - ASSERT(ep->beam[1]); - ep->beam[1] = (BeamInstr) bif_table[i].traced; - } - } - } -} + if (ep->addressv[code_ix] == ep->trampoline.raw) { + continue; + } -static void -reset_bif_trace(void) -{ - int i; - ErtsBpIndex active = erts_active_bp_ix(); - - for (i = 0; i < BIF_SIZE; ++i) { - Export *ep = bif_export[i]; - GenericBp* g = ep->info.u.gen_bp; - if (g && g->data[active].flags == 0) { - if (ExportIsBuiltIn(ep)) { - ASSERT(ep->beam[1]); - ep->beam[1] = (BeamInstr) bif_table[i].f; - } - } + if (BeamIsOpCode(ep->trampoline.op, op_trace_jump_W)) { + ep->trampoline.op = (BeamInstr) 0; + ep->trampoline.trace.address = (BeamInstr) 0; + } } } @@ -1858,6 +1744,8 @@ Eterm erts_seq_trace(Process *p, Eterm arg1, Eterm arg2, if (arg1 == am_send) { current_flag = SEQ_TRACE_SEND; + } else if (arg1 == am_spawn) { + current_flag = SEQ_TRACE_SPAWN; } else if (arg1 == am_receive) { current_flag = SEQ_TRACE_RECEIVE; } else if (arg1 == am_print) { @@ -1976,8 +1864,9 @@ BIF_RETTYPE erl_seq_trace_info(Process *p, Eterm item) } if (have_no_seqtrace(SEQ_TRACE_TOKEN(p))) { - if ((item == am_send) || (item == am_receive) || - (item == am_print) || (item == am_timestamp) + if ((item == am_send) || (item == am_spawn) || + (item == am_receive) || (item == am_print) + || (item == am_timestamp) || (item == am_monotonic_timestamp) || (item == am_strict_monotonic_timestamp)) { hp = HAlloc(p,3); @@ -1992,6 +1881,8 @@ BIF_RETTYPE erl_seq_trace_info(Process *p, Eterm item) if (item == am_send) { current_flag = SEQ_TRACE_SEND; + } else if (item == am_spawn) { + current_flag = SEQ_TRACE_SPAWN; } else if (item == am_receive) { current_flag = SEQ_TRACE_RECEIVE; } else if (item == am_print) { @@ -2041,7 +1932,7 @@ BIF_RETTYPE seq_trace_print_1(BIF_ALIST_1) if (have_no_seqtrace(SEQ_TRACE_TOKEN(BIF_P))) { BIF_RET(am_false); } - seq_trace_update_send(BIF_P); + seq_trace_update_serial(BIF_P); seq_trace_output(SEQ_TRACE_TOKEN(BIF_P), BIF_ARG_1, SEQ_TRACE_PRINT, NIL, BIF_P); BIF_RET(am_true); @@ -2062,7 +1953,7 @@ BIF_RETTYPE seq_trace_print_2(BIF_ALIST_2) } if (!EQ(BIF_ARG_1, SEQ_TRACE_TOKEN_LABEL(BIF_P))) BIF_RET(am_false); - seq_trace_update_send(BIF_P); + seq_trace_update_serial(BIF_P); seq_trace_output(SEQ_TRACE_TOKEN(BIF_P), BIF_ARG_2, SEQ_TRACE_PRINT, NIL, BIF_P); BIF_RET(am_true); diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 5937bd64ec..2bcdb47a54 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -93,11 +93,9 @@ erts_flxctr_dec_read_centralized(&(DB)->common.counters, ERTS_DB_TABLE_NITEMS_COUNTER_ID) #define RESET_NITEMS(DB) \ erts_flxctr_reset(&(DB)->common.counters, ERTS_DB_TABLE_NITEMS_COUNTER_ID) -/* - * The following symbols can be manipulated to "tune" the linear hash array - */ + #define GROW_LIMIT(NACTIVE) ((NACTIVE)*1) -#define SHRINK_LIMIT(NACTIVE) ((NACTIVE) / 2) +#define SHRINK_LIMIT(TB) erts_atomic_read_nob(&(TB)->shrink_limit) /* ** We want the first mandatory segment to be small (to reduce minimal footprint) @@ -137,6 +135,11 @@ #define BUCKET(tb, i) SEGTAB(tb)[SLOT_IX_TO_SEG_IX(i)]->buckets[(i) & EXT_SEGSZ_MASK] +#ifdef DEBUG +# define DBG_BUCKET_INACTIVE ((HashDbTerm*)0xdead5107) +#endif + + /* * When deleting a table, the number of records to delete. * Approximate number, because we must delete entire buckets. @@ -377,7 +380,7 @@ typedef int (*extra_match_validator_t)(int keypos, Eterm match, Eterm guard, Ete */ static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix); static void alloc_seg(DbTableHash *tb); -static int free_seg(DbTableHash *tb, int free_records); +static int free_seg(DbTableHash *tb); static HashDbTerm* next_live(DbTableHash *tb, Uint *iptr, erts_rwmtx_t** lck_ptr, HashDbTerm *list); static HashDbTerm* search_list(DbTableHash* tb, Eterm key, @@ -476,10 +479,8 @@ static int db_raw_next_hash(Process* p, DbTable *tbl, Eterm key, Eterm *ret); static ERTS_INLINE void try_shrink(DbTableHash* tb) { - int nactive = NACTIVE(tb); int nitems = NITEMS(tb); - if (nactive > FIRST_SEGSZ && nitems < SHRINK_LIMIT(nactive) - && !IS_FIXED(tb)) { + if (nitems < SHRINK_LIMIT(tb) && !IS_FIXED(tb)) { shrink(tb, nitems); } } @@ -693,6 +694,7 @@ int db_create_hash(Process *p, DbTable *tbl) erts_atomic_init_nob(&tb->szm, FIRST_SEGSZ_MASK); erts_atomic_init_nob(&tb->nactive, FIRST_SEGSZ); + erts_atomic_init_nob(&tb->shrink_limit, 0); erts_atomic_init_nob(&tb->fixdel, (erts_aint_t)NULL); erts_atomic_init_nob(&tb->segtab, (erts_aint_t)NULL); SET_SEGTAB(tb, tb->first_segtab); @@ -779,7 +781,7 @@ static int db_next_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret) b = next_live(tb, &ix, &lck, b->next); if (tb->common.status & (DB_BAG | DB_DUPLICATE_BAG)) { while (b != 0) { - if (!has_key(tb, b, key, hval) && !is_pseudo_deleted(b)) { + if (!has_key(tb, b, key, hval)) { break; } b = next_live(tb, &ix, &lck, b->next); @@ -789,6 +791,7 @@ static int db_next_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret) *ret = am_EOT; } else { + ASSERT(!is_pseudo_deleted(b)); *ret = db_copy_key(p, tbl, &b->dbterm); RUNLOCK_HASH(lck); } @@ -2474,7 +2477,7 @@ static SWord db_free_table_continue_hash(DbTable *tbl, SWord reds) erts_atomic_set_relb(&tb->fixdel, (erts_aint_t)NULL); while(tb->nslots != 0) { - reds -= EXT_SEGSZ/64 + free_seg(tb, 1); + reds -= EXT_SEGSZ/64 + free_seg(tb); /* * If we have done enough work, get out here. @@ -2672,6 +2675,34 @@ static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix) return est; } +static void calc_shrink_limit(DbTableHash* tb) +{ + erts_aint_t shrink_limit; + + if (tb->nslots >= (FIRST_SEGSZ + 2*EXT_SEGSZ)) { + /* + * Start shrink when we can remove one extra segment + * and still remain below 50% load. + */ + shrink_limit = (tb->nslots - EXT_SEGSZ) / 2; + } + else { + /* + * But don't shrink below two segments. + * Why? In order to have chance of getting rid of the last extra segment, + * and rehash it into the first small segment, we either have to start + * early and do speculative joining of buckets or we have to join a lot + * of buckets during each delete-op. + * + * Instead keep segment #2 once allocated. I also think it's a good bet + * a shrinking large table will grow large again. + */ + shrink_limit = 0; + } + erts_atomic_set_nob(&tb->shrink_limit, shrink_limit); +} + + /* Extend table with one new segment */ static void alloc_seg(DbTableHash *tb) @@ -2690,8 +2721,17 @@ static void alloc_seg(DbTableHash *tb) segtab[seg_ix] = (struct segment*) erts_db_alloc(ERTS_ALC_T_DB_SEG, (DbTable *) tb, SIZEOF_SEGMENT(EXT_SEGSZ)); - sys_memset(segtab[seg_ix], 0, SIZEOF_SEGMENT(EXT_SEGSZ)); +#ifdef DEBUG + { + int i; + for (i = 0; i < EXT_SEGSZ; i++) { + segtab[seg_ix]->buckets[i] = DBG_BUCKET_INACTIVE; + } + } +#endif tb->nslots += EXT_SEGSZ; + + calc_shrink_limit(tb); } static void dealloc_ext_segtab(void* lop_data) @@ -2701,10 +2741,19 @@ static void dealloc_ext_segtab(void* lop_data) erts_free(ERTS_ALC_T_DB_SEG, est); } -/* Shrink table by freeing the top segment +struct dealloc_seg_ops { + struct segment* segp; + Uint seg_sz; + + struct ext_segtab* est; +}; + +/* Shrink table by removing the top segment ** free_records: 1=free any records in segment, 0=assume segment is empty +** ds_ops: (out) Instructions for dealloc_seg(). */ -static int free_seg(DbTableHash *tb, int free_records) +static int remove_seg(DbTableHash *tb, int free_records, + struct dealloc_seg_ops *ds_ops) { const int seg_ix = SLOT_IX_TO_SEG_IX(tb->nslots) - 1; struct segment** const segtab = SEGTAB(tb); @@ -2712,24 +2761,47 @@ static int free_seg(DbTableHash *tb, int free_records) Uint seg_sz; int nrecords = 0; + ERTS_LC_ASSERT(IS_TAB_WLOCKED(tb) || tb->common.status & DB_DELETE + || erts_atomic_read_nob(&tb->is_resizing)); + ASSERT(segp != NULL); -#ifndef DEBUG - if (free_records) -#endif - { - int i = (seg_ix == 0) ? FIRST_SEGSZ : EXT_SEGSZ; - while (i--) { - HashDbTerm* p = segp->buckets[i]; + if (free_records) { + int ix, n; + if (seg_ix == 0) { + /* First segment (always fully active) */ + n = FIRST_SEGSZ; + ix = FIRST_SEGSZ-1; + } + else if (NACTIVE(tb) < tb->nslots) { + /* Last extended segment partially active */ + n = (NACTIVE(tb) - FIRST_SEGSZ) & EXT_SEGSZ_MASK; + ix = (NACTIVE(tb)-1) & EXT_SEGSZ_MASK; + } + else { + /* Full extended segment */ + n = EXT_SEGSZ; + ix = EXT_SEGSZ - 1; + } + for ( ; n > 0; n--, ix--) { + HashDbTerm* p = segp->buckets[ix & EXT_SEGSZ_MASK]; while(p != 0) { HashDbTerm* nxt = p->next; - ASSERT(free_records); /* segment not empty as assumed? */ free_term(tb, p); p = nxt; ++nrecords; } } } - +#ifdef DEBUG + else { + int ix = (seg_ix == 0) ? FIRST_SEGSZ-1 : EXT_SEGSZ-1; + for ( ; ix >= 0; ix--) { + ASSERT(segp->buckets[ix] == DBG_BUCKET_INACTIVE); + } + } +#endif + + ds_ops->est = NULL; if (seg_ix >= NSEG_1) { struct ext_segtab* est = ErtsContainerStruct_(segtab,struct ext_segtab,segtab); @@ -2738,35 +2810,64 @@ static int free_seg(DbTableHash *tb, int free_records) SET_SEGTAB(tb, est->prev_segtab); tb->nsegs = est->prev_nsegs; - if (!tb->common.is_thread_safe) { - /* - * Table is doing a graceful shrink operation and we must avoid - * deallocating this segtab while it may still be read by other - * threads. Schedule deallocation with thread progress to make - * sure no lingering threads are still hanging in BUCKET macro - * with an old segtab pointer. - */ - erts_schedule_db_free(&tb->common, dealloc_ext_segtab, - est, &est->lop, - SIZEOF_EXT_SEGTAB(est->nsegs)); - } - else - erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable*)tb, est, - SIZEOF_EXT_SEGTAB(est->nsegs)); + ds_ops->est = est; } } + seg_sz = (seg_ix == 0) ? FIRST_SEGSZ : EXT_SEGSZ; - erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable *)tb, segp, SIZEOF_SEGMENT(seg_sz)); + tb->nslots -= seg_sz; + ASSERT(tb->nslots >= 0); + + ds_ops->segp = segp; + ds_ops->seg_sz = seg_sz; #ifdef DEBUG if (seg_ix < tb->nsegs) SEGTAB(tb)[seg_ix] = NULL; #endif - tb->nslots -= seg_sz; - ASSERT(tb->nslots >= 0); + calc_shrink_limit(tb); return nrecords; } +/* + * Deallocate segment removed by remove_seg() + */ +static void dealloc_seg(DbTableHash *tb, struct dealloc_seg_ops* ds_ops) +{ + struct ext_segtab* est = ds_ops->est; + + if (est) { + if (!tb->common.is_thread_safe) { + /* + * Table is doing a graceful shrink operation and we must avoid + * deallocating this segtab while it may still be read by other + * threads. Schedule deallocation with thread progress to make + * sure no lingering threads are still hanging in BUCKET macro + * with an old segtab pointer. + */ + erts_schedule_db_free(&tb->common, dealloc_ext_segtab, + est, &est->lop, + SIZEOF_EXT_SEGTAB(est->nsegs)); + } + else + erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable*)tb, est, + SIZEOF_EXT_SEGTAB(est->nsegs)); + } + + erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable *)tb, + ds_ops->segp, SIZEOF_SEGMENT(ds_ops->seg_sz)); +} + +/* Remove and deallocate top segment and all its contained objects */ +static int free_seg(DbTableHash *tb) +{ + struct dealloc_seg_ops ds_ops; + int reds; + + reds = remove_seg(tb, 1, &ds_ops); + dealloc_seg(tb, &ds_ops); + return reds; +} /* ** Copy terms from ptr1 until ptr2 @@ -2888,6 +2989,7 @@ static void grow(DbTableHash* tb, int nitems) pnext = &BUCKET(tb, from_ix); p = *pnext; to_pnext = &BUCKET(tb, to_ix); + ASSERT(*to_pnext == DBG_BUCKET_INACTIVE); while (p != NULL) { if (is_pseudo_deleted(p)) { /* rare but possible with fine locking */ *pnext = p->next; @@ -2924,19 +3026,21 @@ abort: */ static void shrink(DbTableHash* tb, int nitems) { - HashDbTerm** src_bp; - HashDbTerm** dst_bp; + struct dealloc_seg_ops ds_ops; + HashDbTerm* src; + HashDbTerm* tail; HashDbTerm** bp; erts_rwmtx_t* lck; int src_ix, dst_ix, low_szm; int nactive; int loop_limit = 5; + ds_ops.segp = NULL; do { if (!begin_resizing(tb)) return; /* already in progress */ nactive = NACTIVE(tb); - if (!(nactive > FIRST_SEGSZ && nitems < SHRINK_LIMIT(nactive))) { + if (!(nitems < SHRINK_LIMIT(tb))) { goto abort; /* already done (race) */ } src_ix = nactive - 1; @@ -2953,41 +3057,49 @@ static void shrink(DbTableHash* tb, int nitems) goto abort; } - src_bp = &BUCKET(tb, src_ix); - dst_bp = &BUCKET(tb, dst_ix); - bp = src_bp; - - /* - * We join lists by appending "dst" at the end of "src" - * as we must step through "src" anyway to purge pseudo deleted. - */ - while(*bp != NULL) { - if (is_pseudo_deleted(*bp)) { - HashDbTerm* deleted = *bp; - *bp = deleted->next; - free_term(tb, deleted); - } else { - bp = &(*bp)->next; - } - } - *bp = *dst_bp; - *dst_bp = *src_bp; - *src_bp = NULL; - + src = BUCKET(tb, src_ix); +#ifdef DEBUG + BUCKET(tb, src_ix) = DBG_BUCKET_INACTIVE; +#endif nactive = src_ix; erts_atomic_set_nob(&tb->nactive, nactive); if (dst_ix == 0) { erts_atomic_set_relb(&tb->szm, low_szm); } - WUNLOCK_HASH(lck); - if (tb->nslots - src_ix >= EXT_SEGSZ) { - free_seg(tb, 0); + remove_seg(tb, 0, &ds_ops); } done_resizing(tb); - } while (--loop_limit - && nactive > FIRST_SEGSZ && nitems < SHRINK_LIMIT(nactive)); + if (src) { + /* + * We join buckets by appending "dst" list at the end of "src" list + * as we must step through "src" anyway to purge pseudo deleted. + */ + bp = &BUCKET(tb, dst_ix); + tail = *bp; + *bp = src; + + while(*bp != NULL) { + if (is_pseudo_deleted(*bp)) { + HashDbTerm* deleted = *bp; + *bp = deleted->next; + free_term(tb, deleted); + } else { + bp = &(*bp)->next; + } + } + *bp = tail; + } + + WUNLOCK_HASH(lck); + + if (ds_ops.segp) { + dealloc_seg(tb, &ds_ops); + ds_ops.segp = NULL; + } + + } while (--loop_limit && nitems < SHRINK_LIMIT(tb)); return; abort: diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h index 9759d8b466..b26b82056f 100644 --- a/erts/emulator/beam/erl_db_hash.h +++ b/erts/emulator/beam/erl_db_hash.h @@ -63,9 +63,10 @@ typedef struct db_table_hash_fine_locks { typedef struct db_table_hash { DbTableCommon common; - /* SMP: szm and nactive are write-protected by is_resizing or table write lock */ + /* szm, nactive, shrink_limit are write-protected by is_resizing or table write lock */ erts_atomic_t szm; /* current size mask. */ erts_atomic_t nactive; /* Number of "active" slots */ + erts_atomic_t shrink_limit; /* Shrink table when fewer objects than this */ erts_atomic_t segtab; /* The segment table (struct segment**) */ struct segment* first_segtab[1]; diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index ed09a34ae4..096cb8a778 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -2612,7 +2612,10 @@ restart: break; case matchCaller: ASSERT(c_p == self); - if (!(c_p->cp) || !(cp = find_function_from_pc(c_p->cp))) { + t = c_p->stop[0]; + if (is_not_CP(t)) { + *esp++ = am_undefined; + } else if (!(cp = find_function_from_pc(cp_val(t)))) { *esp++ = am_undefined; } else { ehp = HAllocX(build_proc, 4, HEAP_XTRA); @@ -5218,7 +5221,7 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace) Eterm l; Uint32 ret_flags; Uint sz; - BeamInstr *save_cp; + Eterm save_cp; if (trace && !(is_list(against) || against == NIL)) { return THE_NON_VALUE; @@ -5262,13 +5265,13 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace) ++n; l = CDR(list_val(l)); } - save_cp = p->cp; - p->cp = NULL; + save_cp = p->stop[0]; + p->stop[0] = NIL; res = erts_match_set_run_trace(p, p, mps, arr, n, ERTS_PAM_COPY_RESULT|ERTS_PAM_IGNORE_TRACE_SILENT, &ret_flags); - p->cp = save_cp; + p->stop[0] = save_cp; } else { n = 0; arr = NULL; diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index 9c866250bb..79a1fdb8b9 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -100,27 +100,6 @@ int erts_fun_table_sz(void) } ErlFunEntry* -erts_put_fun_entry(Eterm mod, int uniq, int index) -{ - ErlFunEntry template; - ErlFunEntry* fe; - erts_aint_t refc; - ASSERT(is_atom(mod)); - template.old_uniq = uniq; - template.old_index = index; - template.module = mod; - erts_fun_write_lock(); - fe = (ErlFunEntry *) hash_put(&erts_fun_table, (void*) &template); - sys_memset(fe->uniq, 0, sizeof(fe->uniq)); - fe->index = 0; - refc = erts_refc_inctest(&fe->refc, 0); - if (refc < 2) /* New or pending delete */ - erts_refc_inc(&fe->refc, 1); - erts_fun_write_unlock(); - return fe; -} - -ErlFunEntry* erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index, byte* uniq, int index, int arity) { @@ -130,12 +109,12 @@ erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index, ASSERT(is_atom(mod)); template.old_uniq = old_uniq; - template.old_index = old_index; + template.index = index; template.module = mod; erts_fun_write_lock(); fe = (ErlFunEntry *) hash_put(&erts_fun_table, (void*) &template); sys_memcpy(fe->uniq, uniq, sizeof(fe->uniq)); - fe->index = index; + fe->old_index = old_index; fe->arity = arity; refc = erts_refc_inctest(&fe->refc, 0); if (refc < 2) /* New or pending delete */ @@ -144,13 +123,6 @@ erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index, return fe; } -struct my_key { - Eterm mod; - byte* uniq; - int index; - ErlFunEntry* fe; -}; - ErlFunEntry* erts_get_fun_entry(Eterm mod, int uniq, int index) { @@ -159,7 +131,7 @@ erts_get_fun_entry(Eterm mod, int uniq, int index) ASSERT(is_atom(mod)); template.old_uniq = uniq; - template.old_index = index; + template.index = index; template.module = mod; erts_fun_read_lock(); ret = (ErlFunEntry *) hash_get(&erts_fun_table, (void*) &template); @@ -199,36 +171,33 @@ erts_erase_fun_entry(ErlFunEntry* fe) erts_fun_write_unlock(); } +struct fun_purge_foreach_args { + BeamInstr *start; + BeamInstr *end; +}; + +static void fun_purge_foreach(ErlFunEntry *fe, struct fun_purge_foreach_args *arg) +{ + BeamInstr* addr = fe->address; + if (arg->start <= addr && addr < arg->end) { + fe->pend_purge_address = addr; + ERTS_THR_WRITE_MEMORY_BARRIER; + fe->address = unloaded_fun; +#ifdef HIPE + fe->pend_purge_native_address = fe->native_address; + hipe_set_closure_stub(fe); +#endif + erts_purge_state_add_fun(fe); + } +} + void erts_fun_purge_prepare(BeamInstr* start, BeamInstr* end) { - int limit; - HashBucket** bucket; - int i; + struct fun_purge_foreach_args args = {start, end}; erts_fun_read_lock(); - limit = erts_fun_table.size; - bucket = erts_fun_table.bucket; - for (i = 0; i < limit; i++) { - HashBucket* b = bucket[i]; - - while (b) { - ErlFunEntry* fe = (ErlFunEntry *) b; - BeamInstr* addr = fe->address; - - if (start <= addr && addr < end) { - fe->pend_purge_address = addr; - ERTS_THR_WRITE_MEMORY_BARRIER; - fe->address = unloaded_fun; -#ifdef HIPE - fe->pend_purge_native_address = fe->native_address; - hipe_set_closure_stub(fe); -#endif - erts_purge_state_add_fun(fe); - } - b = b->next; - } - } + hash_foreach(&erts_fun_table, (HFOREACH_FUN)fun_purge_foreach, &args); erts_fun_read_unlock(); } @@ -278,36 +247,34 @@ erts_fun_purge_complete(ErlFunEntry **funs, Uint no) ERTS_THR_WRITE_MEMORY_BARRIER; } +struct dump_fun_foreach_args { + fmtfn_t to; + void *to_arg; +}; + +static void +dump_fun_foreach(ErlFunEntry *fe, struct dump_fun_foreach_args *args) +{ + erts_print(args->to, args->to_arg, "=fun\n"); + erts_print(args->to, args->to_arg, "Module: %T\n", fe->module); + erts_print(args->to, args->to_arg, "Uniq: %d\n", fe->old_uniq); + erts_print(args->to, args->to_arg, "Index: %d\n",fe->old_index); + erts_print(args->to, args->to_arg, "Address: %p\n", fe->address); +#ifdef HIPE + erts_print(args->to, args->to_arg, "Native_address: %p\n", fe->native_address); +#endif + erts_print(args->to, args->to_arg, "Refc: %ld\n", erts_refc_read(&fe->refc, 1)); +} + void erts_dump_fun_entries(fmtfn_t to, void *to_arg) { - int limit; - HashBucket** bucket; - int i; + struct dump_fun_foreach_args args = {to, to_arg}; int lock = !ERTS_IS_CRASH_DUMPING; - if (lock) erts_fun_read_lock(); - limit = erts_fun_table.size; - bucket = erts_fun_table.bucket; - for (i = 0; i < limit; i++) { - HashBucket* b = bucket[i]; - - while (b) { - ErlFunEntry* fe = (ErlFunEntry *) b; - erts_print(to, to_arg, "=fun\n"); - erts_print(to, to_arg, "Module: %T\n", fe->module); - erts_print(to, to_arg, "Uniq: %d\n", fe->old_uniq); - erts_print(to, to_arg, "Index: %d\n",fe->old_index); - erts_print(to, to_arg, "Address: %p\n", fe->address); -#ifdef HIPE - erts_print(to, to_arg, "Native_address: %p\n", fe->native_address); -#endif - erts_print(to, to_arg, "Refc: %ld\n", erts_refc_read(&fe->refc, 1)); - b = b->next; - } - } + hash_foreach(&erts_fun_table, (HFOREACH_FUN)dump_fun_foreach, &args); if (lock) erts_fun_read_unlock(); } @@ -315,15 +282,27 @@ erts_dump_fun_entries(fmtfn_t to, void *to_arg) static HashValue fun_hash(ErlFunEntry* obj) { - return (HashValue) (obj->old_uniq ^ obj->old_index ^ atom_val(obj->module)); + return (HashValue) (obj->old_uniq ^ obj->index ^ atom_val(obj->module)); } static int fun_cmp(ErlFunEntry* obj1, ErlFunEntry* obj2) { - return !(obj1->module == obj2->module && + /* + * OTP 23: Use 'index' (instead of 'old_index') when comparing fun + * entries. In OTP 23, multiple make_fun2 instructions may refer to the + * the same 'index' (for the wrapper function generated for the + * 'fun F/A' syntax). + * + * This is safe when loading code compiled with OTP R15 and later, + * because since R15 (2011), the 'index' has been reliably equal + * to 'old_index'. The loader refuses to load modules compiled before + * OTP R15. + */ + + return !(obj1->module == obj2->module && obj1->old_uniq == obj2->old_uniq && - obj1->old_index == obj2->old_index); + obj1->index == obj2->index); } static ErlFunEntry* @@ -333,7 +312,7 @@ fun_alloc(ErlFunEntry* template) sizeof(ErlFunEntry)); obj->old_uniq = template->old_uniq; - obj->old_index = template->old_index; + obj->index = template->index; obj->module = template->module; erts_refc_init(&obj->refc, -1); obj->address = unloaded_fun; diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h index fb2901d866..eefc7a95bb 100644 --- a/erts/emulator/beam/erl_fun.h +++ b/erts/emulator/beam/erl_fun.h @@ -74,7 +74,6 @@ void erts_init_fun_table(void); void erts_fun_info(fmtfn_t, void *); int erts_fun_table_sz(void); -ErlFunEntry* erts_put_fun_entry(Eterm mod, int uniq, int index); ErlFunEntry* erts_get_fun_entry(Eterm mod, int uniq, int index); ErlFunEntry* erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index, diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 13b1f8ab4d..4a6f204cb5 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -65,6 +65,8 @@ # define HARDDEBUG 1 #endif +extern BeamInstr beam_apply[2]; + /* * Returns number of elements in an array. */ @@ -934,13 +936,15 @@ garbage_collect_hibernate(Process* p, int check_long_gc) */ erts_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); ErtsGcQuickSanityCheck(p); - ASSERT(p->stop == p->hend); /* Stack must be empty. */ + ASSERT(p->stop == p->hend - 1); /* Only allow one continuation pointer. */ + ASSERT(p->stop[0] == make_cp(beam_apply+1)); /* * Do it. */ heap_size = p->heap_sz + (p->old_htop - p->old_heap) + p->mbuf_sz; + heap_size += 1; /* Reserve place for continuation pointer */ heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_TMP_HEAP, sizeof(Eterm)*heap_size); @@ -966,13 +970,11 @@ garbage_collect_hibernate(Process* p, int check_long_gc) p->high_water = htop; p->htop = htop; p->hend = p->heap + heap_size; - p->stop = p->hend; + p->stop = p->hend - 1; p->heap_sz = heap_size; heap_size = actual_size = p->htop - p->heap; - if (heap_size == 0) { - heap_size = 1; /* We want a heap... */ - } + heap_size += 1; /* Reserve place for continuation pointer */ FLAGS(p) &= ~F_FORCE_GC; p->live_hf_end = ERTS_INVALID_HFRAG_PTR; @@ -988,14 +990,15 @@ garbage_collect_hibernate(Process* p, int check_long_gc) * hibernated. */ - ASSERT(p->hend - p->stop == 0); /* Empty stack */ ASSERT(actual_size < p->heap_sz); heap = ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*heap_size); sys_memcpy((void *) heap, (void *) p->heap, actual_size*sizeof(Eterm)); ERTS_HEAP_FREE(ERTS_ALC_T_TMP_HEAP, p->heap, p->heap_sz*sizeof(Eterm)); - p->stop = p->hend = heap + heap_size; + p->hend = heap + heap_size; + p->stop = p->hend - 1; + p->stop[0] = make_cp(beam_apply+1); offs = heap - p->heap; area = (char *) p->heap; @@ -2585,7 +2588,7 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) /* * If a NIF or BIF has saved arguments, they need to be added */ - if (erts_setup_nif_export_rootset(p, &roots[n].v, &roots[n].sz)) + if (erts_setup_nfunc_rootset(p, &roots[n].v, &roots[n].sz)) n++; ASSERT(n <= rootset->size); @@ -3233,7 +3236,7 @@ offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size, offset_heap_ptr(objv, nobj, offs, area, area_size); } offset_off_heap(p, offs, area, area_size); - if (erts_setup_nif_export_rootset(p, &v, &sz)) + if (erts_setup_nfunc_rootset(p, &v, &sz)) offset_heap_ptr(v, sz, offs, area, area_size); } diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 547e4064a2..4d0ebbd1ed 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -354,6 +354,7 @@ erl_init(int ncpu, erts_init_bif_chksum(); erts_init_bif_binary(); erts_init_bif_guard(); + erts_init_bif_lists(); erts_init_bif_persistent_term(); erts_init_bif_re(); erts_init_unicode(); /* after RE to get access to PCRE unicode */ diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 1bebf6efe2..42a07a59d6 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -674,7 +674,7 @@ erts_send_message(Process* sender, * Make sure we don't use the heap between those instances. */ if (have_seqtrace(stoken)) { - seq_trace_update_send(sender); + seq_trace_update_serial(sender); seq_trace_output(stoken, message, SEQ_TRACE_SEND, receiver->common.id, sender); diff --git a/erts/emulator/beam/erl_nfunc_sched.c b/erts/emulator/beam/erl_nfunc_sched.c index b8cf2bee0e..8263a6e9b7 100644 --- a/erts/emulator/beam/erl_nfunc_sched.c +++ b/erts/emulator/beam/erl_nfunc_sched.c @@ -30,77 +30,37 @@ #include "erl_nfunc_sched.h" #include "erl_trace.h" -NifExport * -erts_new_proc_nif_export(Process *c_p, int argc) +ErtsNativeFunc * +erts_new_proc_nfunc(Process *c_p, int argc) { + ErtsNativeFunc *nep, *old_nep; size_t size; - int i; - NifExport *nep, *old_nep; - - size = sizeof(NifExport) + (argc-1)*sizeof(Eterm); - nep = erts_alloc(ERTS_ALC_T_NIF_TRAP_EXPORT, size); - for (i = 0; i < ERTS_NUM_CODE_IX; i++) - nep->exp.addressv[i] = &nep->exp.beam[0]; + size = sizeof(ErtsNativeFunc) + (argc-1)*sizeof(Eterm); + nep = erts_alloc(ERTS_ALC_T_NFUNC_TRAP_WRAPPER, size); nep->argc = -1; /* unused marker */ nep->argv_size = argc; - nep->trace = NULL; - old_nep = ERTS_PROC_SET_NIF_TRAP_EXPORT(c_p, nep); + old_nep = ERTS_PROC_SET_NFUNC_TRAP_WRAPPER(c_p, nep); if (old_nep) { - ASSERT(!nep->trace); - erts_free(ERTS_ALC_T_NIF_TRAP_EXPORT, old_nep); + erts_free(ERTS_ALC_T_NFUNC_TRAP_WRAPPER, old_nep); } return nep; } void -erts_destroy_nif_export(Process *p) +erts_destroy_nfunc(Process *p) { - NifExport *nep = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, NULL); + ErtsNativeFunc *nep = ERTS_PROC_SET_NFUNC_TRAP_WRAPPER(p, NULL); if (nep) { if (nep->m) - erts_nif_export_cleanup_nif_mod(nep); - erts_free(ERTS_ALC_T_NIF_TRAP_EXPORT, nep); + erts_nfunc_cleanup_nif_mod(nep); + erts_free(ERTS_ALC_T_NFUNC_TRAP_WRAPPER, nep); } } -void -erts_nif_export_save_trace(Process *c_p, NifExport *nep, int applying, - Export* ep, BeamInstr *cp, Uint32 flags, - Uint32 flags_meta, BeamInstr* I, - ErtsTracer meta_tracer) -{ - NifExportTrace *netp; - ASSERT(nep && nep->argc >= 0); - ASSERT(!nep->trace); - netp = erts_alloc(ERTS_ALC_T_NIF_EXP_TRACE, - sizeof(NifExportTrace)); - netp->applying = applying; - netp->ep = ep; - netp->cp = cp; - netp->flags = flags; - netp->flags_meta = flags_meta; - netp->I = I; - netp->meta_tracer = NIL; - erts_tracer_update(&netp->meta_tracer, meta_tracer); - nep->trace = netp; -} - -void -erts_nif_export_restore_trace(Process *c_p, Eterm result, NifExport *nep) -{ - NifExportTrace *netp = nep->trace; - nep->trace = NULL; - erts_bif_trace_epilogue(c_p, result, netp->applying, netp->ep, - netp->cp, netp->flags, netp->flags_meta, - netp->I, netp->meta_tracer); - erts_tracer_update(&netp->meta_tracer, NIL); - erts_free(ERTS_ALC_T_NIF_EXP_TRACE, netp); -} - -NifExport * -erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc, +ErtsNativeFunc * +erts_nfunc_schedule(Process *c_p, Process *dirty_shadow_proc, ErtsCodeMFA *mfa, BeamInstr *pc, BeamInstr instr, void *dfunc, void *ifunc, @@ -110,7 +70,7 @@ erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc, Process *used_proc; ErtsSchedulerData *esdp; Eterm* reg; - NifExport* nep; + ErtsNativeFunc* nep; int i; ERTS_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) @@ -133,10 +93,10 @@ erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc, reg = esdp->x_reg_array; if (mfa) - nep = erts_get_proc_nif_export(c_p, (int) mfa->arity); + nep = erts_get_proc_nfunc(c_p, (int) mfa->arity); else { /* If no mfa, this is not the first schedule... */ - nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); + nep = ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(c_p); ASSERT(nep && nep->argc >= 0); } @@ -148,16 +108,15 @@ erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc, for (i = 0; i < (int) mfa->arity; i++) nep->argv[i] = reg[i]; nep->pc = pc; - nep->cp = c_p->cp; nep->mfa = mfa; nep->current = c_p->current; ASSERT(argc >= 0); nep->argc = (int) mfa->arity; nep->m = NULL; - ASSERT(!erts_check_nif_export_in_area(c_p, + ASSERT(!erts_check_nfunc_in_area(c_p, (char *) nep, - (sizeof(NifExport) + (sizeof(ErtsNativeFunc) + (sizeof(Eterm) *(nep->argc-1))))); } @@ -167,14 +126,14 @@ erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc, reg[i] = argv[i]; } ASSERT(is_atom(mod) && is_atom(func)); - nep->exp.info.mfa.module = mod; - nep->exp.info.mfa.function = func; - nep->exp.info.mfa.arity = (Uint) argc; - nep->exp.beam[0] = (BeamInstr) instr; /* call_nif || apply_bif */ - nep->exp.beam[1] = (BeamInstr) dfunc; + nep->trampoline.info.mfa.module = mod; + nep->trampoline.info.mfa.function = func; + nep->trampoline.info.mfa.arity = (Uint) argc; + nep->trampoline.call_op = (BeamInstr) instr; /* call_bif || call_nif */ + nep->trampoline.dfunc = (BeamInstr) dfunc; nep->func = ifunc; used_proc->arity = argc; used_proc->freason = TRAP; - used_proc->i = (BeamInstr*) nep->exp.addressv[0]; + used_proc->i = (BeamInstr*)&nep->trampoline.call_op; return nep; } diff --git a/erts/emulator/beam/erl_nfunc_sched.h b/erts/emulator/beam/erl_nfunc_sched.h index 1cb252eba5..4dae242d4f 100644 --- a/erts/emulator/beam/erl_nfunc_sched.h +++ b/erts/emulator/beam/erl_nfunc_sched.h @@ -25,92 +25,78 @@ #include "bif.h" #include "error.h" -typedef struct { - int applying; - Export* ep; - BeamInstr *cp; - Uint32 flags; - Uint32 flags_meta; - BeamInstr* I; - ErtsTracer meta_tracer; -} NifExportTrace; - /* - * NIF exports need a few more items than the Export struct provides, - * including the erl_module_nif* and a NIF function pointer, so the - * NifExport below adds those. The Export member must be first in the - * struct. A number of values are stored for error handling purposes - * only. + * Native function wrappers are used to schedule native functions on both + * normal and dirty schedulers. + * + * A number of values are only stored for error handling, and the fields + * following `current` can be omitted when a wrapper is statically "scheduled" + * through placement in a function stub. * - * 'argc' is >= 0 when NifExport is in use, and < 0 when not. + * 'argc' is >= 0 when ErtsNativeFunc is in use, and < 0 when not. */ typedef struct { - Export exp; + struct { + ErtsCodeInfo info; + BeamInstr call_op; /* call_bif || call_nif */ + BeamInstr dfunc; + } trampoline; + struct erl_module_nif* m; /* NIF module, or NULL if BIF */ void *func; /* Indirect NIF or BIF to execute (may be unused) */ ErtsCodeMFA *current;/* Current as set when originally called */ - NifExportTrace *trace; /* --- The following is only used on error --- */ BeamInstr *pc; /* Program counter */ - BeamInstr *cp; /* Continuation pointer */ ErtsCodeMFA *mfa; /* MFA of original call */ int argc; /* Number of arguments in original call */ int argv_size; /* Allocated size of argv */ Eterm argv[1]; /* Saved arguments from the original call */ -} NifExport; - -NifExport *erts_new_proc_nif_export(Process *c_p, int argc); -void erts_nif_export_save_trace(Process *c_p, NifExport *nep, int applying, - Export* ep, BeamInstr *cp, Uint32 flags, - Uint32 flags_meta, BeamInstr* I, - ErtsTracer meta_tracer); -void erts_nif_export_restore_trace(Process *c_p, Eterm result, NifExport *nep); -void erts_destroy_nif_export(Process *p); -NifExport *erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc, - ErtsCodeMFA *mfa, BeamInstr *pc, - BeamInstr instr, - void *dfunc, void *ifunc, - Eterm mod, Eterm func, - int argc, const Eterm *argv); -void erts_nif_export_cleanup_nif_mod(NifExport *ep); /* erl_nif.c */ -ERTS_GLB_INLINE NifExport *erts_get_proc_nif_export(Process *c_p, int extra); -ERTS_GLB_INLINE int erts_setup_nif_export_rootset(Process* proc, Eterm** objv, - Uint* nobj); -ERTS_GLB_INLINE int erts_check_nif_export_in_area(Process *p, - char *start, Uint size); -ERTS_GLB_INLINE void erts_nif_export_restore(Process *c_p, NifExport *ep, - Eterm result); -ERTS_GLB_INLINE void erts_nif_export_restore_error(Process* c_p, BeamInstr **pc, - Eterm *reg, ErtsCodeMFA **nif_mfa); -ERTS_GLB_INLINE int erts_nif_export_check_save_trace(Process *c_p, Eterm result, - int applying, Export* ep, - BeamInstr *cp, Uint32 flags, - Uint32 flags_meta, BeamInstr* I, - ErtsTracer meta_tracer); +} ErtsNativeFunc; + +ErtsNativeFunc *erts_new_proc_nfunc(Process *c_p, int argc); +void erts_destroy_nfunc(Process *p); +ErtsNativeFunc *erts_nfunc_schedule(Process *c_p, Process *dirty_shadow_proc, + ErtsCodeMFA *mfa, BeamInstr *pc, + BeamInstr instr, + void *dfunc, void *ifunc, + Eterm mod, Eterm func, + int argc, const Eterm *argv); +void erts_nfunc_cleanup_nif_mod(ErtsNativeFunc *ep); /* erl_nif.c */ +ERTS_GLB_INLINE ErtsNativeFunc *erts_get_proc_nfunc(Process *c_p, int extra); +ERTS_GLB_INLINE int erts_setup_nfunc_rootset(Process* proc, Eterm** objv, + Uint* nobj); +ERTS_GLB_INLINE int erts_check_nfunc_in_area(Process *p, + char *start, Uint size); +ERTS_GLB_INLINE void erts_nfunc_restore(Process *c_p, ErtsNativeFunc *ep, + Eterm result); +ERTS_GLB_INLINE void erts_nfunc_restore_error(Process* c_p, + BeamInstr **pc, + Eterm *reg, + ErtsCodeMFA **nif_mfa); ERTS_GLB_INLINE Process *erts_proc_shadow2real(Process *c_p); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE NifExport * -erts_get_proc_nif_export(Process *c_p, int argc) +ERTS_GLB_INLINE ErtsNativeFunc * +erts_get_proc_nfunc(Process *c_p, int argc) { - NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); + ErtsNativeFunc *nep = ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(c_p); if (!nep || (nep->argc < 0 && nep->argv_size < argc)) - return erts_new_proc_nif_export(c_p, argc); + return erts_new_proc_nfunc(c_p, argc); return nep; } /* * If a process has saved arguments, they need to be part of the GC * rootset. The function below is called from setup_rootset() in - * erl_gc.c. Any exception term saved in the NifExport is also made + * erl_gc.c. Any exception term saved in the ErtsNativeFunc is also made * part of the GC rootset here; it always resides in rootset[0]. */ ERTS_GLB_INLINE int -erts_setup_nif_export_rootset(Process* proc, Eterm** objv, Uint* nobj) +erts_setup_nfunc_rootset(Process* proc, Eterm** objv, Uint* nobj) { - NifExport* ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + ErtsNativeFunc* ep = (ErtsNativeFunc*) ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(proc); if (!ep || ep->argc <= 0) return 0; @@ -121,18 +107,16 @@ erts_setup_nif_export_rootset(Process* proc, Eterm** objv, Uint* nobj) } /* - * Check if nif export points into code area... + * Check if native func wrapper points into code area... */ ERTS_GLB_INLINE int -erts_check_nif_export_in_area(Process *p, char *start, Uint size) +erts_check_nfunc_in_area(Process *p, char *start, Uint size) { - NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(p); + ErtsNativeFunc *nep = ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(p); if (!nep || nep->argc < 0) return 0; if (ErtsInArea(nep->pc, start, size)) return 1; - if (ErtsInArea(nep->cp, start, size)) - return 1; if (ErtsInArea(nep->mfa, start, size)) return 1; if (ErtsInArea(nep->current, start, size)) @@ -141,7 +125,7 @@ erts_check_nif_export_in_area(Process *p, char *start, Uint size) } ERTS_GLB_INLINE void -erts_nif_export_restore(Process *c_p, NifExport *ep, Eterm result) +erts_nfunc_restore(Process *c_p, ErtsNativeFunc *ep, Eterm result) { ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())); ERTS_LC_ASSERT(!(c_p->static_flags @@ -151,43 +135,21 @@ erts_nif_export_restore(Process *c_p, NifExport *ep, Eterm result) c_p->current = ep->current; ep->argc = -1; /* Unused nif-export marker... */ - if (ep->trace) - erts_nif_export_restore_trace(c_p, result, ep); } ERTS_GLB_INLINE void -erts_nif_export_restore_error(Process* c_p, BeamInstr **pc, +erts_nfunc_restore_error(Process* c_p, BeamInstr **pc, Eterm *reg, ErtsCodeMFA **nif_mfa) { - NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); + ErtsNativeFunc *nep = (ErtsNativeFunc *) ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(c_p); int ix; ASSERT(nep); *pc = nep->pc; - c_p->cp = nep->cp; *nif_mfa = nep->mfa; for (ix = 0; ix < nep->argc; ix++) reg[ix] = nep->argv[ix]; - erts_nif_export_restore(c_p, nep, THE_NON_VALUE); -} - -ERTS_GLB_INLINE int -erts_nif_export_check_save_trace(Process *c_p, Eterm result, - int applying, Export* ep, - BeamInstr *cp, Uint32 flags, - Uint32 flags_meta, BeamInstr* I, - ErtsTracer meta_tracer) -{ - if (is_non_value(result) && c_p->freason == TRAP) { - NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); - if (nep && nep->argc >= 0) { - erts_nif_export_save_trace(c_p, nep, applying, ep, - cp, flags, flags_meta, - I, meta_tracer); - return 1; - } - } - return 0; + erts_nfunc_restore(c_p, nep, THE_NON_VALUE); } ERTS_GLB_INLINE Process * @@ -210,10 +172,10 @@ erts_proc_shadow2real(Process *c_p) #if defined(ERTS_WANT_NFUNC_SCHED_INTERNALS__) && !defined(ERTS_NFUNC_SCHED_INTERNALS__) #define ERTS_NFUNC_SCHED_INTERNALS__ -#define ERTS_I_BEAM_OP_TO_NIF_EXPORT(I) \ - (ASSERT(BeamIsOpCode(*(I), op_apply_bif) || \ - BeamIsOpCode(*(I), op_call_nif)), \ - ((NifExport *) (((char *) (I)) - offsetof(NifExport, exp.beam[0])))) +#define ERTS_I_BEAM_OP_TO_NFUNC(I) \ + (ASSERT(BeamIsOpCode(*(I), op_call_bif_W) || \ + BeamIsOpCode(*(I), op_call_nif_WWW)), \ + ((ErtsNativeFunc *) (((char *) (I)) - offsetof(ErtsNativeFunc, trampoline.call_op)))) #include "erl_message.h" diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 1fbe362330..6e27b4b7cb 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -309,10 +309,10 @@ void erts_post_nif(ErlNifEnv* env) /* - * Initialize a NifExport struct. Create it if needed and store it in the + * Initialize a ErtsNativeFunc struct. Create it if needed and store it in the * proc. The direct_fp function is what will be invoked by op_call_nif, and * the indirect_fp function, if not NULL, is what the direct_fp function - * will call. If the allocated NifExport isn't enough to hold all of argv, + * will call. If the allocated ErtsNativeFunc isn't enough to hold all of argv, * allocate a larger one. Save 'current' and registers if first time this * call is scheduled. */ @@ -321,7 +321,7 @@ static ERTS_INLINE ERL_NIF_TERM schedule(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp, Eterm mod, Eterm func_name, int argc, const ERL_NIF_TERM argv[]) { - NifExport *ep; + ErtsNativeFunc *ep; Process *c_p, *dirty_shadow_proc; execution_state(env, &c_p, NULL); @@ -332,10 +332,10 @@ schedule(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp, ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p)); - ep = erts_nif_export_schedule(c_p, dirty_shadow_proc, + ep = erts_nfunc_schedule(c_p, dirty_shadow_proc, c_p->current, - c_p->cp, - BeamOpCodeAddr(op_call_nif), + cp_val(c_p->stop[0]), + BeamOpCodeAddr(op_call_nif_WWW), direct_fp, indirect_fp, mod, func_name, argc, (const Eterm *) argv); @@ -356,7 +356,7 @@ erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm * { int exiting; ERL_NIF_TERM *argv = (ERL_NIF_TERM *) reg; - NifExport *nep = ERTS_I_BEAM_OP_TO_NIF_EXPORT(I); + ErtsNativeFunc *nep = ERTS_I_BEAM_OP_TO_NFUNC(I); ErtsCodeMFA *codemfa = erts_code_to_codemfa(I); NativeFunPtr dirty_nif = (NativeFunPtr) I[1]; ErlNifEnv env; @@ -364,7 +364,7 @@ erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm * #ifdef DEBUG erts_aint32_t state = erts_atomic32_read_nob(&c_p->state); - ASSERT(nep == ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p)); + ASSERT(nep == ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(c_p)); ASSERT(!c_p->scheduler_data); ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING) @@ -815,7 +815,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, } #endif if (have_seqtrace(stoken)) { - seq_trace_update_send(c_p); + seq_trace_update_serial(c_p); seq_trace_output(stoken, msg, SEQ_TRACE_SEND, rp->common.id, c_p); } @@ -2823,7 +2823,7 @@ int enif_consume_timeslice(ErlNifEnv* env, int percent) } static ERTS_INLINE void -nif_export_cleanup_nif_mod(NifExport *ep) +nfunc_cleanup_nif_mod(ErtsNativeFunc *ep) { if (erts_refc_dectest(&ep->m->rt_dtor_cnt, 0) == 0 && ep->m->mod == NULL) close_lib(ep->m); @@ -2831,17 +2831,17 @@ nif_export_cleanup_nif_mod(NifExport *ep) } void -erts_nif_export_cleanup_nif_mod(NifExport *ep) +erts_nfunc_cleanup_nif_mod(ErtsNativeFunc *ep) { - nif_export_cleanup_nif_mod(ep); + nfunc_cleanup_nif_mod(ep); } static ERTS_INLINE void -nif_export_restore(Process *c_p, NifExport *ep, Eterm res) +nfunc_restore(Process *c_p, ErtsNativeFunc *ep, Eterm res) { - erts_nif_export_restore(c_p, ep, res); + erts_nfunc_restore(c_p, ep, res); ASSERT(ep->m); - nif_export_cleanup_nif_mod(ep); + nfunc_cleanup_nif_mod(ep); } @@ -2858,15 +2858,15 @@ static ERL_NIF_TERM dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { Process* proc; - NifExport* ep; + ErtsNativeFunc* ep; execution_state(env, &proc, NULL); ASSERT(argc == 1); ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc))); - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + ep = (ErtsNativeFunc*) ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(proc); ASSERT(ep); - nif_export_restore(proc, ep, argv[0]); + nfunc_restore(proc, ep, argv[0]); return argv[0]; } @@ -2878,21 +2878,22 @@ dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { ERL_NIF_TERM ret; Process* proc; - NifExport* ep; + ErtsNativeFunc* ep; Eterm exception; execution_state(env, &proc, NULL); ASSERT(argc == 1); ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc))); - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + ep = (ErtsNativeFunc*) ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(proc); ASSERT(ep); exception = argv[0]; /* argv overwritten by restore below... */ - nif_export_cleanup_nif_mod(ep); + nfunc_cleanup_nif_mod(ep); ret = enif_raise_exception(env, exception); - /* Restore orig info for error and clear nif export in handle_error() */ - proc->freason |= EXF_RESTORE_NIF; + /* Restore orig info for error and clear native func wrapper in + * handle_error() */ + proc->freason |= EXF_RESTORE_NFUNC; return ret; } @@ -2929,7 +2930,7 @@ static_schedule_dirty_nif(ErlNifEnv* env, erts_aint32_t dirty_psflg, int argc, const ERL_NIF_TERM argv[]) { Process *proc; - NifExport *ep; + ErtsNativeFunc *ep; Eterm mod, func; NativeFunPtr fp; @@ -2939,12 +2940,11 @@ static_schedule_dirty_nif(ErlNifEnv* env, erts_aint32_t dirty_psflg, * Called in order to schedule statically determined * dirty NIF calls... * - * Note that 'current' does not point into a NifExport - * structure; only a structure with similar - * parts (located in code). + * Note that 'current' does not point into a ErtsNativeFunc + * structure; only a structure with similar parts (located in code). */ - ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); + ep = ErtsContainerStruct(proc->current, ErtsNativeFunc, trampoline.info.mfa); mod = proc->current->module; func = proc->current->function; fp = (NativeFunPtr) ep->func; @@ -2983,12 +2983,12 @@ execute_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { Process* proc; NativeFunPtr fp; - NifExport* ep; + ErtsNativeFunc* ep; ERL_NIF_TERM result; execution_state(env, &proc, NULL); - ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); + ep = ErtsContainerStruct(proc->current, ErtsNativeFunc, trampoline.info.mfa); fp = ep->func; ASSERT(ep); ASSERT(!env->exception_thrown); @@ -3001,20 +3001,20 @@ execute_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) result = (*fp)(env, argc, argv); - ASSERT(ep == ERTS_PROC_GET_NIF_TRAP_EXPORT(proc)); + ASSERT(ep == ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(proc)); if (is_value(result) || proc->freason != TRAP) { /* Done (not rescheduled)... */ ASSERT(ep->func == ERTS_DBG_NIF_NOT_SCHED_MARKER); if (!env->exception_thrown) - nif_export_restore(proc, ep, result); + nfunc_restore(proc, ep, result); else { - nif_export_cleanup_nif_mod(ep); + nfunc_cleanup_nif_mod(ep); /* * Restore orig info for error and clear nif * export in handle_error() */ - proc->freason |= EXF_RESTORE_NIF; + proc->freason |= EXF_RESTORE_NFUNC; } } @@ -4117,8 +4117,23 @@ static struct erl_module_nif* create_lib(const ErlNifEntry* src) return lib; }; +/* load_nif/2 is implemented as an instruction as it needs to know where it + * was called from, and it's a pain to get that information in a BIF. + * + * This is a small stub that rejects apply(erlang, load_nif, [Path, Args]). */ +BIF_RETTYPE load_nif_2(BIF_ALIST_2) { + if (BIF_P->flags & F_HIPE_MODE) { + BIF_RET(load_nif_error(BIF_P, "notsup", + "Calling load_nif from HiPE compiled modules " + "not supported")); + } + + BIF_RET(load_nif_error(BIF_P, "bad_lib", + "load_nif/2 must be explicitly called from the NIF " + "module. It cannot be called through apply/3.")); +} -BIF_RETTYPE load_nif_2(BIF_ALIST_2) +Eterm erts_load_nif(Process *c_p, BeamInstr *I, Eterm filename, Eterm args) { static const char bad_lib[] = "bad_lib"; static const char upgrade[] = "upgrade"; @@ -4141,41 +4156,25 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) struct erl_module_instance* this_mi; struct erl_module_instance* prev_mi; - if (BIF_P->flags & F_HIPE_MODE) { - ret = load_nif_error(BIF_P, "notsup", "Calling load_nif from HiPE compiled " - "modules not supported"); - BIF_RET(ret); - } - encoding = erts_get_native_filename_encoding(); if (encoding == ERL_FILENAME_WIN_WCHAR) { /* Do not convert the lib name to utf-16le yet, do that in win32 specific code */ /* since lib_name is used in error messages */ encoding = ERL_FILENAME_UTF8; } - lib_name = erts_convert_filename_to_encoding(BIF_ARG_1, NULL, 0, + lib_name = erts_convert_filename_to_encoding(filename, NULL, 0, ERTS_ALC_T_TMP, 1, 0, encoding, NULL, 0); if (!lib_name) { - BIF_ERROR(BIF_P, BADARG); - } - - if (!erts_try_seize_code_write_permission(BIF_P)) { - erts_free(ERTS_ALC_T_TMP, lib_name); - ERTS_BIF_YIELD2(bif_export[BIF_load_nif_2], - BIF_P, BIF_ARG_1, BIF_ARG_2); + return THE_NON_VALUE; } /* Block system (is this the right place to do it?) */ - erts_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); erts_thr_progress_block(); /* Find calling module */ - ASSERT(BIF_P->current != NULL); - ASSERT(BIF_P->current->module == am_erlang - && BIF_P->current->function == am_load_nif - && BIF_P->current->arity == 2); - caller = find_function_from_pc(BIF_P->cp); + caller = find_function_from_pc(I); ASSERT(caller != NULL); mod_atom = caller->module; ASSERT(is_atom(mod_atom)); @@ -4195,7 +4194,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) this_mi = &module_p->curr; prev_mi = &module_p->old; if (in_area(caller, module_p->old.code_hdr, module_p->old.code_length)) { - ret = load_nif_error(BIF_P, "old_code", "Calling load_nif from old " + ret = load_nif_error(c_p, "old_code", "Calling load_nif from old " "module '%T' not allowed", mod_atom); goto error; } else if (module_p->on_load) { @@ -4209,52 +4208,52 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) } if (this_mi->nif != NULL) { - ret = load_nif_error(BIF_P,"reload","NIF library already loaded" + ret = load_nif_error(c_p,"reload","NIF library already loaded" " (reload disallowed since OTP 20)."); } else if (init_func == NULL && (err=erts_sys_ddll_open(lib_name, &handle, &errdesc)) != ERL_DE_NO_ERROR) { const char slogan[] = "Failed to load NIF library"; if (strstr(errdesc.str, lib_name) != NULL) { - ret = load_nif_error(BIF_P, "load_failed", "%s: '%s'", slogan, errdesc.str); + ret = load_nif_error(c_p, "load_failed", "%s: '%s'", slogan, errdesc.str); } else { - ret = load_nif_error(BIF_P, "load_failed", "%s %s: '%s'", slogan, lib_name, errdesc.str); + ret = load_nif_error(c_p, "load_failed", "%s %s: '%s'", slogan, lib_name, errdesc.str); } } else if (init_func == NULL && erts_sys_ddll_load_nif_init(handle, &init_func, &errdesc) != ERL_DE_NO_ERROR) { - ret = load_nif_error(BIF_P, bad_lib, "Failed to find library init" + ret = load_nif_error(c_p, bad_lib, "Failed to find library init" " function: '%s'", errdesc.str); } else if ((taint ? erts_add_taint(mod_atom) : 0, (entry = erts_sys_ddll_call_nif_init(init_func)) == NULL)) { - ret = load_nif_error(BIF_P, bad_lib, "Library init-call unsuccessful"); + ret = load_nif_error(c_p, bad_lib, "Library init-call unsuccessful"); } else if (entry->major > ERL_NIF_MAJOR_VERSION || (entry->major == ERL_NIF_MAJOR_VERSION && entry->minor > ERL_NIF_MINOR_VERSION)) { char* fmt = "That '%T' NIF library needs %s or newer. Either try to" " recompile the NIF lib or use a newer erts runtime."; - ret = load_nif_error(BIF_P, bad_lib, fmt, mod_atom, entry->min_erts); + ret = load_nif_error(c_p, bad_lib, fmt, mod_atom, entry->min_erts); } else if (entry->major < ERL_NIF_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD || (entry->major==2 && entry->minor == 5)) { /* experimental maps */ char* fmt = "That old NIF library (%d.%d) is not compatible with this " "erts runtime (%d.%d). Try recompile the NIF lib."; - ret = load_nif_error(BIF_P, bad_lib, fmt, entry->major, entry->minor, + ret = load_nif_error(c_p, bad_lib, fmt, entry->major, entry->minor, ERL_NIF_MAJOR_VERSION, ERL_NIF_MINOR_VERSION); } else if (AT_LEAST_VERSION(entry, 2, 1) && sys_strcmp(entry->vm_variant, ERL_NIF_VM_VARIANT) != 0) { - ret = load_nif_error(BIF_P, bad_lib, "Library (%s) not compiled for " + ret = load_nif_error(c_p, bad_lib, "Library (%s) not compiled for " "this vm variant (%s).", entry->vm_variant, ERL_NIF_VM_VARIANT); } else if (!erts_is_atom_str((char*)entry->name, mod_atom, 1)) { - ret = load_nif_error(BIF_P, bad_lib, "Library module name '%s' does not" + ret = load_nif_error(c_p, bad_lib, "Library module name '%s' does not" " match calling module '%T'", entry->name, mod_atom); } else { @@ -4273,7 +4272,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) if (!erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1) || (ci_pp = get_func_pp(this_mi->code_hdr, f_atom, f->arity))==NULL) { - ret = load_nif_error(BIF_P,bad_lib,"Function not found %T:%s/%u", + ret = load_nif_error(c_p,bad_lib,"Function not found %T:%s/%u", mod_atom, f->name, f->arity); } else if (f->flags) { @@ -4285,16 +4284,13 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) * a load error. */ if (f->flags != ERL_NIF_DIRTY_JOB_IO_BOUND && f->flags != ERL_NIF_DIRTY_JOB_CPU_BOUND) - ret = load_nif_error(BIF_P, bad_lib, "Illegal flags field value %d for NIF %T:%s/%u", + ret = load_nif_error(c_p, bad_lib, "Illegal flags field value %d for NIF %T:%s/%u", f->flags, mod_atom, f->name, f->arity); } - else if (erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0]) - < BEAM_NIF_MIN_FUNC_SZ) - { - ret = load_nif_error(BIF_P,bad_lib,"No explicit call to load_nif" - " in module (%T:%s/%u too small)", - mod_atom, f->name, f->arity); - } + + ASSERT(erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0]) + >= BEAM_NATIVE_MIN_FUNC_SZ); + /*erts_fprintf(stderr, "Found NIF %T:%s/%u\r\n", mod_atom, f->name, f->arity);*/ } @@ -4313,23 +4309,23 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) if (prev_mi->nif != NULL) { /**************** Upgrade ***************/ void* prev_old_data = prev_mi->nif->priv_data; if (entry->upgrade == NULL) { - ret = load_nif_error(BIF_P, upgrade, "Upgrade not supported by this NIF library."); + ret = load_nif_error(c_p, upgrade, "Upgrade not supported by this NIF library."); goto error; } - erts_pre_nif(&env, BIF_P, lib, NULL); - veto = entry->upgrade(&env, &lib->priv_data, &prev_mi->nif->priv_data, BIF_ARG_2); + erts_pre_nif(&env, c_p, lib, NULL); + veto = entry->upgrade(&env, &lib->priv_data, &prev_mi->nif->priv_data, args); erts_post_nif(&env); if (veto) { prev_mi->nif->priv_data = prev_old_data; - ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful (%d).", veto); + ret = load_nif_error(c_p, upgrade, "Library upgrade-call unsuccessful (%d).", veto); } } else if (entry->load != NULL) { /********* Initial load ***********/ - erts_pre_nif(&env, BIF_P, lib, NULL); - veto = entry->load(&env, &lib->priv_data, BIF_ARG_2); + erts_pre_nif(&env, c_p, lib, NULL); + veto = entry->load(&env, &lib->priv_data, args); erts_post_nif(&env); if (veto) { - ret = load_nif_error(BIF_P, "load", "Library load-call unsuccessful (%d).", veto); + ret = load_nif_error(c_p, "load", "Library load-call unsuccessful (%d).", veto); } } if (ret == am_ok) { @@ -4351,12 +4347,12 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) code_ptr = erts_codeinfo_to_code(ci); if (ci->u.gen_bp == NULL) { - code_ptr[0] = BeamOpCodeAddr(op_call_nif); + code_ptr[0] = BeamOpCodeAddr(op_call_nif_WWW); } else { /* Function traced, patch the original instruction word */ GenericBp* g = ci->u.gen_bp; ASSERT(BeamIsOpCode(code_ptr[0], op_i_generic_breakpoint)); - g->orig_instr = BeamOpCodeAddr(op_call_nif); + g->orig_instr = BeamOpCodeAddr(op_call_nif_WWW); } if (f->flags) { code_ptr[3] = (BeamInstr) f->fptr; @@ -4383,8 +4379,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) } erts_thr_progress_unblock(); - erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_release_code_write_permission(); + erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); erts_free(ERTS_ALC_T_TMP, lib_name); BIF_RET(ret); diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 8863e219e2..ae3193be81 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -977,7 +977,7 @@ static void print_node(void *venp, void *vpndp) if(pndp->sysname == NIL) { erts_print(pndp->to, pndp->to_arg, "Name: %T ", enp->sysname); } - erts_print(pndp->to, pndp->to_arg, " %d", enp->creation); + erts_print(pndp->to, pndp->to_arg, " %u", enp->creation); #ifdef DEBUG erts_print(pndp->to, pndp->to_arg, " (refc=%ld)", erts_refc_read(&enp->refc, 0)); @@ -1020,7 +1020,7 @@ void erts_print_node_info(fmtfn_t to, /* ----------------------------------------------------------------------- */ void -erts_set_this_node(Eterm sysname, Uint creation) +erts_set_this_node(Eterm sysname, Uint32 creation) { ERTS_LC_ASSERT(erts_thr_progress_is_blocking()); ASSERT(2 <= de_refc_read(erts_this_dist_entry, 2)); diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index ffaafbbbea..beae2df75f 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -264,7 +264,7 @@ void erts_set_dist_entry_pending(DistEntry *); void erts_set_dist_entry_connected(DistEntry *, Eterm, Uint); ErlNode *erts_find_or_insert_node(Eterm, Uint32, Eterm); void erts_schedule_delete_node(ErlNode *); -void erts_set_this_node(Eterm, Uint); +void erts_set_this_node(Eterm, Uint32); Uint erts_node_table_size(void); void erts_init_node_tables(int); void erts_node_table_info(fmtfn_t, void *); diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c index d5e0e3b218..b60fb64342 100644 --- a/erts/emulator/beam/erl_proc_sig_queue.c +++ b/erts/emulator/beam/erl_proc_sig_queue.c @@ -995,7 +995,7 @@ send_gen_exit_signal(Process *c_p, Eterm from_tag, seq_trace = c_p && have_seqtrace(token); if (seq_trace) - seq_trace_update_send(c_p); + seq_trace_update_serial(c_p); #ifdef USE_VM_PROBES utag_sz = 0; diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index ac0a4f8902..b62ec77d65 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -708,10 +708,10 @@ erts_pre_init_process(void) erts_psd_required_locks[ERTS_PSD_DELAYED_GC_TASK_QS].set_locks = ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS; - erts_psd_required_locks[ERTS_PSD_NIF_TRAP_EXPORT].get_locks - = ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS; - erts_psd_required_locks[ERTS_PSD_NIF_TRAP_EXPORT].set_locks - = ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS; + erts_psd_required_locks[ERTS_PSD_NFUNC_TRAP_WRAPPER].get_locks + = ERTS_PSD_NFUNC_TRAP_WRAPPER_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_NFUNC_TRAP_WRAPPER].set_locks + = ERTS_PSD_NFUNC_TRAP_WRAPPER_SET_LOCKS; erts_psd_required_locks[ERTS_PSD_ETS_OWNED_TABLES].get_locks = ERTS_PSD_ETS_OWNED_TABLES_GET_LOCKS; @@ -6478,8 +6478,8 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, ASSERT(!(state & (ERTS_PSFLG_DIRTY_IO_PROC |ERTS_PSFLG_DIRTY_CPU_PROC)) - || (BeamIsOpCode(*p->i, op_call_nif) - || BeamIsOpCode(*p->i, op_apply_bif))); + || (BeamIsOpCode(*p->i, op_call_nif_WWW) + || BeamIsOpCode(*p->i, op_call_bif_W))); a = state; @@ -10992,8 +10992,13 @@ erts_set_gc_state(Process *c_p, int enable) ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(c_p)); if (!enable) { - c_p->flags |= F_DISABLE_GC; - return 0; + /* Strictly speaking it's not illegal to disable the GC when it's + * already disabled, but we risk enabling the GC prematurely if (for + * example) a BIF were to blindly disable it when trapping and then + * re-enable it before returning its result. */ + ASSERT(!(c_p->flags & F_DISABLE_GC)); + c_p->flags |= F_DISABLE_GC; + return 0; } c_p->flags &= ~F_DISABLE_GC; @@ -11453,7 +11458,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #else arg_size = size_object_litopt(args, &litarea); #endif - heap_need = arg_size; + heap_need = arg_size + 1; /* Reserve place for continuation pointer */ p->flags = flags; @@ -11502,7 +11507,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->old_hend = p->old_htop = p->old_heap = NULL; p->high_water = p->heap; p->gen_gcs = 0; - p->stop = p->hend = p->heap + sz; + p->hend = p->heap + sz; + p->stop = p->hend - 1; /* Reserve place for continuation pointer */ p->htop = p->heap; p->heap_sz = sz; p->abandoned_heap = NULL; @@ -11520,7 +11526,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->current = &p->u.initial; p->i = (BeamInstr *) beam_apply; - p->cp = (BeamInstr *) beam_apply+1; + p->stop[0] = make_cp(beam_apply + 1); p->arg_reg = p->def_arg_reg; p->max_arg_reg = sizeof(p->def_arg_reg)/sizeof(p->def_arg_reg[0]); @@ -11583,9 +11589,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->mbuf_sz = 0; erts_atomic_init_nob(&p->psd, (erts_aint_t) NULL); p->dictionary = NULL; - p->seq_trace_lastcnt = 0; - p->seq_trace_clock = 0; - SEQ_TRACE_TOKEN(p) = NIL; #ifdef USE_VM_PROBES DT_UTAG(p) = NIL; DT_UTAG_FLAGS(p) = 0; @@ -11606,6 +11609,45 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->fp_exception = 0; #endif + /* seq_trace is handled before regular tracing as the latter may touch the + * trace token. */ + if (have_seqtrace(SEQ_TRACE_TOKEN(parent))) { + Eterm token; + Uint token_sz; + Eterm *hp; + + ASSERT(SEQ_TRACE_TOKEN_ARITY(parent) == 5); + ASSERT(is_immed(SEQ_TRACE_TOKEN_FLAGS(parent))); + ASSERT(is_immed(SEQ_TRACE_TOKEN_SERIAL(parent))); + ASSERT(is_immed(SEQ_TRACE_TOKEN_LASTCNT(parent))); + + seq_trace_update_serial(parent); + + token = SEQ_TRACE_TOKEN(parent); + token_sz = size_object(token); + + hp = HAlloc(p, token_sz); + SEQ_TRACE_TOKEN(p) = copy_struct(token, token_sz, &hp, &MSO(p)); + + /* The counters behave the same way on spawning as they do on messages; + * we don't inherit our parent's lastcnt. */ + p->seq_trace_lastcnt = parent->seq_trace_clock; + p->seq_trace_clock = parent->seq_trace_clock; + + ASSERT((locks & (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE)) == + (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE)); + + locks &= ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + erts_proc_unlock(parent, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + + seq_trace_output(token, NIL, SEQ_TRACE_SPAWN, p->common.id, parent); + } else { + SEQ_TRACE_TOKEN(p) = NIL; + p->seq_trace_lastcnt = 0; + p->seq_trace_clock = 0; + } + if (IS_TRACED(parent)) { if (ERTS_TRACE_FLAGS(parent) & F_TRACE_SOS) { ERTS_TRACE_FLAGS(p) |= (ERTS_TRACE_FLAGS(parent) & TRACEE_FLAGS); @@ -11627,9 +11669,14 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). } } if (ARE_TRACE_FLAGS_ON(parent, F_TRACE_PROCS)) { - locks &= ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); - erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); - erts_proc_unlock(parent, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + /* The locks may already be released if seq_trace is enabled as + * well. */ + if ((locks & (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE)) + == (ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE)) { + locks &= ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + erts_proc_unlock(parent, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + } trace_proc_spawn(parent, am_spawn, p->common.id, mod, func, args); if (so->flags & SPO_LINK) trace_proc(parent, locks, parent, am_link, p->common.id); @@ -11793,7 +11840,6 @@ void erts_init_empty_process(Process *p) p->u.initial.function = 0; p->u.initial.arity = 0; p->catches = 0; - p->cp = NULL; p->i = NULL; p->current = NULL; @@ -11871,7 +11917,6 @@ erts_debug_verify_clean_empty_process(Process* p) ASSERT(p->bif_timers == NULL); ASSERT(p->dictionary == NULL); ASSERT(p->catches == 0); - ASSERT(p->cp == NULL); ASSERT(p->i == NULL); ASSERT(p->current == NULL); @@ -11931,7 +11976,7 @@ delete_process(Process* p) if (pbt) erts_free(ERTS_ALC_T_BPD, (void *) pbt); - erts_destroy_nif_export(p); + erts_destroy_nfunc(p); /* Cleanup psd */ @@ -13117,9 +13162,6 @@ erts_program_counter_info(fmtfn_t to, void *to_arg, Process *p) erts_print(to, to_arg, "Program counter: %p (", p->i); print_function_from_pc(to, to_arg, p->i); erts_print(to, to_arg, ")\n"); - erts_print(to, to_arg, "CP: %p (", p->cp); - print_function_from_pc(to, to_arg, p->cp); - erts_print(to, to_arg, ")\n"); state = erts_atomic32_read_acqb(&p->state); if (!(state & (ERTS_PSFLG_RUNNING | ERTS_PSFLG_RUNNING_SYS @@ -13396,9 +13438,6 @@ static void print_current_process_info(fmtfn_t to, void *to_arg, erts_print(to, to_arg, "Current Process Program counter: %p (", p->i); print_function_from_pc(to, to_arg, p->i); erts_print(to, to_arg, ")\n"); - erts_print(to, to_arg, "Current Process CP: %p (", p->cp); - print_function_from_pc(to, to_arg, p->cp); - erts_print(to, to_arg, ")\n"); /* Getting this stacktrace can segfault if we are very very unlucky if called while a process is being garbage collected. diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 5886b576e0..09a6c0e961 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -812,7 +812,7 @@ erts_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi) #define ERTS_PSD_SCHED_ID 2 #define ERTS_PSD_CALL_TIME_BP 3 #define ERTS_PSD_DELAYED_GC_TASK_QS 4 -#define ERTS_PSD_NIF_TRAP_EXPORT 5 +#define ERTS_PSD_NFUNC_TRAP_WRAPPER 5 #define ERTS_PSD_ETS_OWNED_TABLES 6 #define ERTS_PSD_ETS_FIXED_TABLES 7 #define ERTS_PSD_DIST_ENTRY 8 @@ -849,8 +849,8 @@ typedef struct { #define ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS ERTS_PROC_LOCK_MAIN #define ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS ERTS_PROC_LOCK_MAIN -#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ERTS_PROC_LOCK_MAIN -#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ERTS_PROC_LOCK_MAIN +#define ERTS_PSD_NFUNC_TRAP_WRAPPER_GET_LOCKS ERTS_PROC_LOCK_MAIN +#define ERTS_PSD_NFUNC_TRAP_WRAPPER_SET_LOCKS ERTS_PROC_LOCK_MAIN #define ERTS_PSD_ETS_OWNED_TABLES_GET_LOCKS ERTS_PROC_LOCK_STATUS #define ERTS_PSD_ETS_OWNED_TABLES_SET_LOCKS ERTS_PROC_LOCK_STATUS @@ -975,7 +975,6 @@ struct process { unsigned max_arg_reg; /* Maximum number of argument registers available. */ Eterm def_arg_reg[6]; /* Default array for argument registers. */ - BeamInstr* cp; /* (untagged) Continuation pointer (for threaded code). */ BeamInstr* i; /* Program counter for threaded code. */ Sint catches; /* Number of catches on stack */ Sint fcalls; /* @@ -1489,6 +1488,8 @@ extern int erts_system_profile_ts_type; #define SEQ_TRACE_SEND (1 << 0) #define SEQ_TRACE_RECEIVE (1 << 1) #define SEQ_TRACE_PRINT (1 << 2) +/* (This three-bit gap contains the timestamp.) */ +#define SEQ_TRACE_SPAWN (1 << 6) #define ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT 3 @@ -2037,10 +2038,10 @@ erts_psd_set(Process *p, int ix, void *data) #define ERTS_PROC_SET_DELAYED_GC_TASK_QS(P, PBT) \ ((ErtsProcSysTaskQs *) erts_psd_set((P), ERTS_PSD_DELAYED_GC_TASK_QS, (void *) (PBT))) -#define ERTS_PROC_GET_NIF_TRAP_EXPORT(P) \ - erts_psd_get((P), ERTS_PSD_NIF_TRAP_EXPORT) -#define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, NTE) \ - erts_psd_set((P), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE)) +#define ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(P) \ + erts_psd_get((P), ERTS_PSD_NFUNC_TRAP_WRAPPER) +#define ERTS_PROC_SET_NFUNC_TRAP_WRAPPER(P, NTE) \ + erts_psd_set((P), ERTS_PSD_NFUNC_TRAP_WRAPPER, (void *) (NTE)) #define ERTS_PROC_GET_DIST_ENTRY(P) \ ((DistEntry *) erts_psd_get((P), ERTS_PSD_DIST_ENTRY)) diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index f6f177887c..5c46a10d64 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -830,7 +830,7 @@ trace_receive(Process* receiver, } int -seq_trace_update_send(Process *p) +seq_trace_update_serial(Process *p) { ErtsTracer seq_tracer = erts_get_system_seq_tracer(); ASSERT((is_tuple(SEQ_TRACE_TOKEN(p)) || is_nil(SEQ_TRACE_TOKEN(p)))); @@ -898,6 +898,7 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, switch (type) { case SEQ_TRACE_SEND: type_atom = am_send; break; + case SEQ_TRACE_SPAWN: type_atom = am_spawn; break; case SEQ_TRACE_PRINT: type_atom = am_print; break; case SEQ_TRACE_RECEIVE: type_atom = am_receive; break; default: diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h index af38ef52db..c0f31e0cb6 100644 --- a/erts/emulator/beam/erl_trace.h +++ b/erts/emulator/beam/erl_trace.h @@ -142,12 +142,6 @@ void monitor_generic(Process *p, Eterm type, Eterm spec); Uint erts_trace_flag2bit(Eterm flag); int erts_trace_flags(Eterm List, Uint *pMask, ErtsTracer *pTracer, int *pCpuTimestamp); -Eterm erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr *I); -Eterm -erts_bif_trace_epilogue(Process *p, Eterm result, int applying, - Export* ep, BeamInstr *cp, Uint32 flags, - Uint32 flags_meta, BeamInstr* I, - ErtsTracer meta_tracer); void erts_send_pending_trace_msgs(ErtsSchedulerData *esdp); #define ERTS_CHK_PEND_TRACE_MSGS(ESDP) \ @@ -163,7 +157,9 @@ seq_trace_output_generic((token), (msg), (type), (receiver), NULL, (exitfrom)) void seq_trace_output_generic(Eterm token, Eterm msg, Uint type, Eterm receiver, Process *process, Eterm exitfrom); -int seq_trace_update_send(Process *process); +/* Bump the sequence number if tracing is enabled; must be used before sending + * send/spawn trace messages. */ +int seq_trace_update_serial(Process *process); Eterm erts_seq_trace(Process *process, Eterm atom_type, Eterm atom_true_or_false, diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index 430ac305c5..449243a9b7 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -70,6 +70,7 @@ int erts_fit_in_bits_uint(Uint); Sint erts_list_length(Eterm); int erts_is_builtin(Eterm, Eterm, int); Uint32 make_hash2(Eterm); +Uint32 trapping_make_hash2(Eterm, Eterm*, struct process*); Uint32 make_hash(Eterm); Uint32 make_internal_hash(Eterm, Uint32 salt); diff --git a/erts/emulator/beam/erlang_dtrace.d b/erts/emulator/beam/erlang_dtrace.d index 8792138d53..8864a8ec84 100644 --- a/erts/emulator/beam/erlang_dtrace.d +++ b/erts/emulator/beam/erlang_dtrace.d @@ -176,7 +176,7 @@ provider erlang { * Fired whenever a user function returns. * * @param p the PID (string form) of the process - * @param mfa the m:f/a of the function + * @param mfa the m:f/a of the function being returned from * @param depth the stack depth */ probe function__return(char *p, char *mfa, int depth); @@ -193,7 +193,7 @@ provider erlang { * Fired whenever a Built In Function returns. * * @param p the PID (string form) of the process - * @param mfa the m:f/a of the function + * @param mfa the m:f/a of the function being returned from */ probe bif__return(char *p, char *mfa); @@ -209,7 +209,7 @@ provider erlang { * Fired whenever a Native Function returns. * * @param p the PID (string form) of the process - * @param mfa the m:f/a of the function + * @param mfa the m:f/a of the function being returned from */ probe nif__return(char *p, char *mfa); diff --git a/erts/emulator/beam/error.h b/erts/emulator/beam/error.h index 64c08b1570..44a9809a18 100644 --- a/erts/emulator/beam/error.h +++ b/erts/emulator/beam/error.h @@ -66,13 +66,13 @@ #define EXF_OFFSET EXTAG_BITS #define EXF_BITS 7 -#define EXF_PANIC (1<<(0+EXF_OFFSET)) /* ignore catches */ -#define EXF_THROWN (1<<(1+EXF_OFFSET)) /* nonlocal return */ -#define EXF_LOG (1<<(2+EXF_OFFSET)) /* write to logger on termination */ -#define EXF_NATIVE (1<<(3+EXF_OFFSET)) /* occurred in native code */ -#define EXF_SAVETRACE (1<<(4+EXF_OFFSET)) /* save stack trace in internal form */ -#define EXF_ARGLIST (1<<(5+EXF_OFFSET)) /* has arglist for top of trace */ -#define EXF_RESTORE_NIF (1<<(6+EXF_OFFSET)) /* restore original bif/nif */ +#define EXF_PANIC (1<<(0+EXF_OFFSET)) /* ignore catches */ +#define EXF_THROWN (1<<(1+EXF_OFFSET)) /* nonlocal return */ +#define EXF_LOG (1<<(2+EXF_OFFSET)) /* write to logger on termination */ +#define EXF_NATIVE (1<<(3+EXF_OFFSET)) /* occurred in native code */ +#define EXF_SAVETRACE (1<<(4+EXF_OFFSET)) /* save stack trace in internal form */ +#define EXF_ARGLIST (1<<(5+EXF_OFFSET)) /* has arglist for top of trace */ +#define EXF_RESTORE_NFUNC (1<<(6+EXF_OFFSET)) /* restore original bif/nif */ #define EXC_FLAGBITS (((1<<(EXF_BITS+EXF_OFFSET))-1) \ & ~((1<<(EXF_OFFSET))-1)) diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 946ffeffb8..ca16bfd20e 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -129,14 +129,17 @@ export_alloc(struct export_entry* tmpl_e) obj->info.mfa.module = tmpl->info.mfa.module; obj->info.mfa.function = tmpl->info.mfa.function; obj->info.mfa.arity = tmpl->info.mfa.arity; - obj->beam[0] = 0; + obj->bif_table_index = -1; + obj->is_bif_traced = 0; + + memset(&obj->trampoline, 0, sizeof(obj->trampoline)); + if (BeamOpsAreInitialized()) { - obj->beam[0] = BeamOpCodeAddr(op_call_error_handler); + obj->trampoline.op = BeamOpCodeAddr(op_call_error_handler); } - obj->beam[1] = 0; for (ix=0; ix<ERTS_NUM_CODE_IX; ix++) { - obj->addressv[ix] = obj->beam; + obj->addressv[ix] = obj->trampoline.raw; blob->entryv[ix].slot.index = -1; blob->entryv[ix].ep = &blob->exp; @@ -196,6 +199,19 @@ init_export_table(void) } } +static struct export_entry* init_template(struct export_templ* templ, + Eterm m, Eterm f, unsigned a) +{ + templ->entry.ep = &templ->exp; + templ->entry.slot.index = -1; + templ->exp.info.mfa.module = m; + templ->exp.info.mfa.function = f; + templ->exp.info.mfa.arity = a; + templ->exp.bif_table_index = -1; + templ->exp.is_bif_traced = 0; + return &templ->entry; +} + /* * Return a pointer to the export entry for the given function, * or NULL otherwise. Notes: @@ -214,41 +230,15 @@ erts_find_export_entry(Eterm m, Eterm f, unsigned int a,ErtsCodeIndex code_ix); Export* erts_find_export_entry(Eterm m, Eterm f, unsigned int a, ErtsCodeIndex code_ix) { - HashValue hval = EXPORT_HASH((BeamInstr) m, (BeamInstr) f, (BeamInstr) a); - int ix; - HashBucket* b; - - ix = hval % export_tables[code_ix].htable.size; - b = export_tables[code_ix].htable.bucket[ix]; - - /* - * Note: We have inlined the code from hash.c for speed. - */ - - while (b != (HashBucket*) 0) { - Export* ep = ((struct export_entry*) b)->ep; - if (ep->info.mfa.module == m && - ep->info.mfa.function == f && - ep->info.mfa.arity == a) { - return ep; - } - b = b->next; - } + struct export_templ templ; + struct export_entry *ee = + hash_fetch(&export_tables[code_ix].htable, + init_template(&templ, m, f, a), + (H_FUN)export_hash, (HCMP_FUN)export_cmp); + if (ee) return ee->ep; return NULL; } -static struct export_entry* init_template(struct export_templ* templ, - Eterm m, Eterm f, unsigned a) -{ - templ->entry.ep = &templ->exp; - templ->entry.slot.index = -1; - templ->exp.info.mfa.module = m; - templ->exp.info.mfa.function = f; - templ->exp.info.mfa.arity = a; - return &templ->entry; -} - - /* * Find the export entry for a loaded function. * Returns a NULL pointer if the given function is not loaded, or @@ -268,8 +258,8 @@ erts_find_function(Eterm m, Eterm f, unsigned int a, ErtsCodeIndex code_ix) ee = hash_get(&export_tables[code_ix].htable, init_template(&templ, m, f, a)); if (ee == NULL || - (ee->ep->addressv[code_ix] == ee->ep->beam && - ! BeamIsOpCode(ee->ep->beam[0], op_i_generic_breakpoint))) { + (ee->ep->addressv[code_ix] == ee->ep->trampoline.raw && + ! BeamIsOpCode(ee->ep->trampoline.op, op_i_generic_breakpoint))) { return NULL; } return ee->ep; diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h index ae8dfa4cf8..0190624f79 100644 --- a/erts/emulator/beam/export.h +++ b/erts/emulator/beam/export.h @@ -31,24 +31,72 @@ typedef struct export { - void* addressv[ERTS_NUM_CODE_IX]; /* Pointer to code for function. */ - - ErtsCodeInfo info; /* MUST be just before beam[] */ - - /* - * beam[0]: This entry is 0 unless the 'addressv' field points to it. - * Threaded code instruction to load function - * (em_call_error_handler), execute BIF (em_apply_bif), - * or a breakpoint instruction (op_i_generic_breakpoint). - * beam[1]: Function pointer to BIF function (for BIFs only), - * or pointer to threaded code if the module has an - * on_load function that has not been run yet, or pointer - * to code if function beam[0] is a breakpoint instruction. - * Otherwise: 0. - */ - BeamInstr beam[2]; + /* Pointer to code for function. */ + void* addressv[ERTS_NUM_CODE_IX]; + + /* Index into bif_table[], or -1 if not a BIF. */ + int bif_table_index; + /* Non-zero if this is a BIF that's traced. */ + int is_bif_traced; + + /* This is a small trampoline function that can be used for lazy code + * loading, global call tracing, and so on. It's only valid when + * addressv points to it and should otherwise be left zeroed. + * + * Needless to say, the order of the fields below is significant. */ + ErtsCodeInfo info; + union { + BeamInstr op; /* Union discriminant. */ + + struct { + BeamInstr op; /* op_i_generic_breakpoint */ + BeamInstr address; /* Address of the original function */ + } breakpoint; + + /* This is used when a module refers to (imports) a function that + * hasn't been loaded yet. Upon loading we create an export entry which + * redirects to the error_handler so that the appropriate module will + * be loaded when called (or crash). + * + * This is also used when a module has an on_load callback as we need + * to defer all calls until the callback returns. `deferred` contains + * the address of the original function in this case, and there's an + * awkward condiditon where `deferred` may be set while op is zero. See + * erlang:finish_after_on_load/2 for details. */ + struct { + BeamInstr op; /* op_call_error_handler, or 0 during the last + * phase of code loading when on_load is + * present. See above. */ + BeamInstr deferred; + } not_loaded; + + struct { + BeamInstr op; /* op_trace_jump_W */ + BeamInstr address; /* Address of the traced function */ + } trace; + + BeamInstr raw[2]; /* For use in address comparisons, should not + * be tampered directly. */ + } trampoline; } Export; +#ifdef DEBUG +#define DBG_CHECK_EXPORT(EP, CX) \ + do { \ + if((EP)->addressv[CX] == (EP)->trampoline.raw) { \ + /* The entry currently points at the trampoline, so the + * instructions must be valid. */ \ + ASSERT(((BeamIsOpCode((EP)->trampoline.op, op_i_generic_breakpoint)) && \ + (EP)->trampoline.breakpoint.address != 0) || \ + ((BeamIsOpCode((EP)->trampoline.op, op_trace_jump_W)) && \ + (EP)->trampoline.trace.address != 0) || \ + /* (EP)->trampoline.not_loaded.deferred may be zero. */ \ + (BeamIsOpCode((EP)->trampoline.op, op_call_error_handler))); \ + } \ + } while(0) +#else +#define DBG_CHECK_EXPORT(EP, CX) ((void)(EP), (void)(CX)) +#endif void init_export_table(void); void export_info(fmtfn_t, void *); @@ -71,9 +119,6 @@ extern erts_mtx_t export_staging_lock; #define export_staging_unlock() erts_mtx_unlock(&export_staging_lock) #include "beam_load.h" /* For em_* extern declarations */ -#define ExportIsBuiltIn(EntryPtr) \ -(((EntryPtr)->addressv[erts_active_code_ix()] == (EntryPtr)->beam) && \ - (BeamIsOpCode((EntryPtr)->beam[0], op_apply_bif))) #if ERTS_GLB_INLINE_INCL_FUNC_DEF diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 39bbf62eae..a575e1d743 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -51,18 +51,17 @@ #define MAX_STRING_LEN 0xffff -/* MAX value for the creation field in pid, port and reference - for the local node and for the current external format. - - Larger creation values than this are allowed in external pid, port and refs - encoded with NEW_PID_EXT, NEW_PORT_EXT and NEWER_REFERENCE_EXT. - The point here is to prepare for future upgrade to 32-bit creation. - OTP-19 (erts-8.0) can handle big creation values from other (newer) nodes, - but do not use big creation values for the local node yet, - as we still may have to communicate with older nodes. +/* + * MAX value for the creation field in pid, port and reference + * for the old PID_EXT, PORT_EXT, REFERENCE_EXT and NEW_REFERENCE_EXT. + * Older nodes (OTP 19-22) will send us these so we must be able to decode them. + * + * From OTP 23 DFLAG_BIG_CREATION is mandatory so this node will always + * encode with new big 32-bit creations using NEW_PID_EXT, NEW_PORT_EXT + * and NEWER_REFERENCE_EXT. */ -#define ERTS_MAX_LOCAL_CREATION (3) -#define is_valid_creation(Cre) ((unsigned)(Cre) <= ERTS_MAX_LOCAL_CREATION) +#define ERTS_MAX_TINY_CREATION (3) +#define is_tiny_creation(Cre) ((unsigned)(Cre) <= ERTS_MAX_TINY_CREATION) #undef ERTS_DEBUG_USE_DIST_SEP #ifdef DEBUG @@ -2469,7 +2468,8 @@ enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags) Eterm sysname = ((is_internal_pid(pid) && (dflags & DFLAG_INTERNAL_TAGS)) ? INTERNAL_LOCAL_SYSNAME : pid_node_name(pid)); Uint32 creation = pid_creation(pid); - byte* tagp = ep++; + + *ep++ = NEW_PID_EXT; /* insert atom here containing host and sysname */ ep = enc_atom(acmp, sysname, ep, dflags); @@ -2481,15 +2481,8 @@ enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags) ep += 4; put_int32(os, ep); ep += 4; - if (creation <= ERTS_MAX_LOCAL_CREATION) { - *tagp = PID_EXT; - *ep++ = creation; - } else { - ASSERT(is_external_pid(pid)); - *tagp = NEW_PID_EXT; - put_int32(creation, ep); - ep += 4; - } + put_int32(creation, ep); + ep += 4; return ep; } @@ -2609,7 +2602,7 @@ dec_pid(ErtsDistExternal *edep, ErtsHeapFactory* factory, byte* ep, if (tag == PID_EXT) { cre = get_int8(ep); ep += 1; - if (!is_valid_creation(cre)) { + if (!is_tiny_creation(cre)) { return NULL; } } else { @@ -2870,25 +2863,18 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Eterm sysname = (((dflags & DFLAG_INTERNAL_TAGS) && is_internal_ref(obj)) ? INTERNAL_LOCAL_SYSNAME : ref_node_name(obj)); Uint32 creation = ref_creation(obj); - byte* tagp = ep++; ASSERT(dflags & DFLAG_EXTENDED_REFERENCES); erts_magic_ref_save_bin(obj); + *ep++ = NEWER_REFERENCE_EXT; i = ref_no_numbers(obj); put_int16(i, ep); ep += 2; ep = enc_atom(acmp, sysname, ep, dflags); - if (creation <= ERTS_MAX_LOCAL_CREATION) { - *tagp = NEW_REFERENCE_EXT; - *ep++ = creation; - } else { - ASSERT(is_external_ref(obj)); - *tagp = NEWER_REFERENCE_EXT; - put_int32(creation, ep); - ep += 4; - } + put_int32(creation, ep); + ep += 4; ref_num = ref_numbers(obj); for (j = 0; j < i; j++) { put_int32(ref_num[j], ep); @@ -2901,21 +2887,14 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Eterm sysname = (((dflags & DFLAG_INTERNAL_TAGS) && is_internal_port(obj)) ? INTERNAL_LOCAL_SYSNAME : port_node_name(obj)); Uint32 creation = port_creation(obj); - byte* tagp = ep++; + *ep++ = NEW_PORT_EXT; ep = enc_atom(acmp, sysname, ep, dflags); j = port_number(obj); put_int32(j, ep); ep += 4; - if (creation <= ERTS_MAX_LOCAL_CREATION) { - *tagp = PORT_EXT; - *ep++ = creation; - } else { - ASSERT(is_external_port(obj)); - *tagp = NEW_PORT_EXT; - put_int32(creation, ep); - ep += 4; - } + put_int32(creation, ep); + ep += 4; break; } case LIST_DEF: @@ -3610,7 +3589,7 @@ dec_term_atom_common: if (tag == PORT_EXT) { cre = get_int8(ep); ep++; - if (!is_valid_creation(cre)) { + if (!is_tiny_creation(cre)) { goto error; } } @@ -3657,7 +3636,7 @@ dec_term_atom_common: cre = get_int8(ep); ep += 1; - if (!is_valid_creation(cre)) { + if (!is_tiny_creation(cre)) { goto error; } goto ref_ext_common; @@ -3671,7 +3650,7 @@ dec_term_atom_common: cre = get_int8(ep); ep += 1; - if (!is_valid_creation(cre)) { + if (!is_tiny_creation(cre)) { goto error; } r0 = get_int32(ep); @@ -4066,73 +4045,6 @@ dec_term_atom_common: next = &(funp->creator); break; } - case FUN_EXT: - { - ErlFunThing* funp = (ErlFunThing *) hp; - Eterm module; - Sint old_uniq; - Sint old_index; - unsigned num_free; - int i; - Eterm temp; - - num_free = get_int32(ep); - ep += 4; - hp += ERL_FUN_SIZE; - hp += num_free; - factory->hp = hp; - funp->thing_word = HEADER_FUN; - funp->num_free = num_free; - *objp = make_fun(funp); - - /* Creator pid */ - if ((*ep != PID_EXT && *ep != NEW_PID_EXT) - || (ep = dec_pid(edep, factory, ep+1, - &funp->creator, *ep))==NULL) { - goto error; - } - - /* Module */ - if ((ep = dec_atom(edep, ep, &module)) == NULL) { - goto error; - } - - /* Index */ - if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) { - goto error; - } - if (!is_small(temp)) { - goto error; - } - old_index = unsigned_val(temp); - - /* Uniq */ - if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) { - goto error; - } - if (!is_small(temp)) { - goto error; - } - - /* - * It is safe to link the fun into the fun list only when - * no more validity tests can fail. - */ - funp->next = factory->off_heap->first; - factory->off_heap->first = (struct erl_off_heap_header*)funp; - old_uniq = unsigned_val(temp); - - funp->fe = erts_put_fun_entry(module, old_uniq, old_index); - funp->arity = funp->fe->address[-1] - num_free; - hp = factory->hp; - - /* Environment */ - for (i = num_free-1; i >= 0; i--) { - funp->env[i] = (Eterm) next; - next = funp->env + i; - } - break; - } case ATOM_INTERNAL_REF2: n = get_int16(ep); ep += 2; @@ -4401,30 +4313,21 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, result += 1 + 4 + 1 + i; /* tag,size,sign,digits */ break; case EXTERNAL_PID_DEF: - if (external_pid_creation(obj) > ERTS_MAX_LOCAL_CREATION) - result += 3; - /*fall through*/ case PID_DEF: result += (1 + encode_size_struct2(acmp, pid_node_name(obj), dflags) + - 4 + 4 + 1); + 4 + 4 + 4); break; case EXTERNAL_REF_DEF: - if (external_ref_creation(obj) > ERTS_MAX_LOCAL_CREATION) - result += 3; - /*fall through*/ case REF_DEF: ASSERT(dflags & DFLAG_EXTENDED_REFERENCES); i = ref_no_numbers(obj); result += (1 + 2 + encode_size_struct2(acmp, ref_node_name(obj), dflags) + - 1 + 4*i); + 4 + 4*i); break; case EXTERNAL_PORT_DEF: - if (external_port_creation(obj) > ERTS_MAX_LOCAL_CREATION) - result += 3; - /*fall through*/ case PORT_DEF: result += (1 + encode_size_struct2(acmp, port_node_name(obj), dflags) + - 4 + 1); + 4 + 4); break; case LIST_DEF: { int is_str = is_external_string(obj, &m); @@ -4891,9 +4794,6 @@ init_done: total_size = get_int32(ep); CHKSIZE(total_size); ep += 1+16+4+4; - /*FALLTHROUGH*/ - - case FUN_EXT: CHKSIZE(4); num_free = get_int32(ep); ep += 4; @@ -4904,6 +4804,12 @@ init_done: heap_size += ERL_FUN_SIZE + num_free; break; } + case FUN_EXT: + /* + * OTP 23: No longer support decoding the old fun + * representation. + */ + goto error; case ATOM_INTERNAL_REF2: SKIP(2+atom_extra_skip); atom_extra_skip = 0; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 40c65461bc..b86709b093 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -122,6 +122,10 @@ void erts_nif_demonitored(ErtsResource* resource); extern void erts_add_taint(Eterm mod_atom); extern Eterm erts_nif_taints(Process* p); extern void erts_print_nif_taints(fmtfn_t to, void* to_arg); + +/* Loads the specified NIF. The caller must have code write permission. */ +Eterm erts_load_nif(Process *c_p, BeamInstr *I, Eterm filename, Eterm args); + void erts_unload_nif(struct erl_module_nif* nif); extern void erl_nif_init(void); extern int erts_nif_get_funcs(struct erl_module_nif*, @@ -885,6 +889,8 @@ void erts_bif_info_init(void); /* bif.c */ +void erts_write_bif_wrapper(Export *export, BeamInstr *address); + void erts_queue_monitor_message(Process *, ErtsProcLocks*, Eterm, @@ -906,6 +912,9 @@ Eterm erts_trapping_length_1(Process* p, Eterm* args); Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2); +/* beam_bif_lists.c */ +void erts_init_bif_lists(void); + /* beam_bif_load.c */ Eterm erts_check_process_code(Process *c_p, Eterm module, int *redsp, int fcalls); Eterm erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed); @@ -1147,6 +1156,7 @@ void erts_dirty_process_main(ErtsSchedulerData *); Eterm build_stacktrace(Process* c_p, Eterm exc); Eterm expand_error_value(Process* c_p, Uint freason, Eterm Value); void erts_save_stacktrace(Process* p, struct StackTrace* s, int depth); +BeamInstr *erts_printable_return_address(Process* p, Eterm *E) ERTS_NOINLINE; /* erl_init.c */ diff --git a/erts/emulator/beam/hash.c b/erts/emulator/beam/hash.c index 8954dbb06c..177b7cc3d1 100644 --- a/erts/emulator/beam/hash.c +++ b/erts/emulator/beam/hash.c @@ -30,37 +30,19 @@ #include "hash.h" /* -** List of sizes (all are primes) -*/ -static const int h_size_table[] = { - 2, 5, 11, 23, 47, 97, 197, 397, 797, /* double upto here */ - 1201, 1597, - 2411, 3203, - 4813, 6421, - 9643, 12853, - 19289, 25717, - 51437, - 102877, - 205759, - 411527, - 823117, - 1646237, - 3292489, - 6584983, - 13169977, - 26339969, - 52679969, - -1 -}; - -/* ** Get info about hash ** */ +#define MAX_SHIFT (ERTS_SIZEOF_TERM * 8) + +static int hash_get_slots(Hash *h) { + return UWORD_CONSTANT(1) << (MAX_SHIFT - h->shift); +} + void hash_get_info(HashInfo *hi, Hash *h) { - int size = h->size; + int size = hash_get_slots(h); int i; int max_depth = 0; int objects = 0; @@ -84,7 +66,7 @@ void hash_get_info(HashInfo *hi, Hash *h) ASSERT(objects == h->nobjs); hi->name = h->name; - hi->size = h->size; + hi->size = hash_get_slots(h); hi->used = used; hi->objs = h->nobjs; hi->depth = max_depth; @@ -118,15 +100,15 @@ hash_table_sz(Hash *h) int i; for(i=0;h->name[i];i++); i++; - return sizeof(Hash) + h->size*sizeof(HashBucket*) + i; + return sizeof(Hash) + hash_get_slots(h)*sizeof(HashBucket*) + i; } static ERTS_INLINE void set_thresholds(Hash* h) { - h->grow_threshold = (8*h->size)/5; /* grow at 160% load */ - if (h->size_ix > h->min_size_ix) - h->shrink_threshold = h->size / 5; /* shrink at 20% load */ + h->grow_threshold = (8*hash_get_slots(h))/5; /* grow at 160% load */ + if (h->shift < h->max_shift) + h->shrink_threshold = hash_get_slots(h) / 5; /* shrink at 20% load */ else h->shrink_threshold = -1; /* never shrink below inital size */ } @@ -138,29 +120,27 @@ static ERTS_INLINE void set_thresholds(Hash* h) Hash* hash_init(int type, Hash* h, char* name, int size, HashFunctions fun) { int sz; - int ix = 0; + int shift = 1; h->meta_alloc_type = type; - while (h_size_table[ix] != -1 && h_size_table[ix] < size) - ix++; - if (h_size_table[ix] == -1) - return NULL; - - size = h_size_table[ix]; - sz = size*sizeof(HashBucket*); + while ((UWORD_CONSTANT(1) << shift) < size) + shift++; - h->bucket = (HashBucket**) fun.meta_alloc(h->meta_alloc_type, sz); - - memzero(h->bucket, sz); h->is_allocated = 0; h->name = name; h->fun = fun; - h->size = size; - h->size_ix = ix; - h->min_size_ix = ix; + h->shift = MAX_SHIFT - shift; + h->max_shift = h->shift; h->nobjs = 0; set_thresholds(h); + + sz = hash_get_slots(h) * sizeof(HashBucket*); + h->bucket = (HashBucket**) fun.meta_alloc(h->meta_alloc_type, sz); + memzero(h->bucket, sz); + + ASSERT(h->shift > 0 && h->shift < 64); + return h; } @@ -183,7 +163,7 @@ Hash* hash_new(int type, char* name, int size, HashFunctions fun) */ void hash_delete(Hash* h) { - int old_size = h->size; + int old_size = hash_get_slots(h); int i; for (i = 0; i < old_size; i++) { @@ -206,22 +186,20 @@ void hash_delete(Hash* h) static void rehash(Hash* h, int grow) { int sz; - int old_size = h->size; + int old_size = hash_get_slots(h); HashBucket** new_bucket; int i; if (grow) { - if ((h_size_table[h->size_ix+1]) == -1) - return; - h->size_ix++; + h->shift--; } else { - if (h->size_ix == 0) + if (h->shift == h->max_shift) return; - h->size_ix--; + h->shift++; } - h->size = h_size_table[h->size_ix]; - sz = h->size*sizeof(HashBucket*); + + sz = hash_get_slots(h)*sizeof(HashBucket*); new_bucket = (HashBucket **) h->fun.meta_alloc(h->meta_alloc_type, sz); memzero(new_bucket, sz); @@ -230,7 +208,7 @@ static void rehash(Hash* h, int grow) HashBucket* b = h->bucket[i]; while (b != (HashBucket*) 0) { HashBucket* b_next = b->next; - int ix = b->hvalue % h->size; + Uint ix = hash_get_slot(h, b->hvalue); b->next = new_bucket[ix]; new_bucket[ix] = b; b = b_next; @@ -247,16 +225,7 @@ static void rehash(Hash* h, int grow) */ void* hash_get(Hash* h, void* tmpl) { - HashValue hval = h->fun.hash(tmpl); - int ix = hval % h->size; - HashBucket* b = h->bucket[ix]; - - while(b != (HashBucket*) 0) { - if ((b->hvalue == hval) && (h->fun.cmp(tmpl, (void*)b) == 0)) - return (void*) b; - b = b->next; - } - return (void*) 0; + return hash_fetch(h, tmpl, h->fun.hash, h->fun.cmp); } /* @@ -265,7 +234,7 @@ void* hash_get(Hash* h, void* tmpl) void* hash_put(Hash* h, void* tmpl) { HashValue hval = h->fun.hash(tmpl); - int ix = hval % h->size; + Uint ix = hash_get_slot(h, hval); HashBucket* b = h->bucket[ix]; while(b != (HashBucket*) 0) { @@ -291,7 +260,7 @@ void* hash_put(Hash* h, void* tmpl) void* hash_erase(Hash* h, void* tmpl) { HashValue hval = h->fun.hash(tmpl); - int ix = hval % h->size; + Uint ix = hash_get_slot(h, hval); HashBucket* b = h->bucket[ix]; HashBucket* prev = 0; @@ -323,7 +292,7 @@ void * hash_remove(Hash *h, void *tmpl) { HashValue hval = h->fun.hash(tmpl); - int ix = hval % h->size; + Uint ix = hash_get_slot(h, hval); HashBucket *b = h->bucket[ix]; HashBucket *prev = NULL; @@ -343,11 +312,11 @@ hash_remove(Hash *h, void *tmpl) return NULL; } -void hash_foreach(Hash* h, void (*func)(void *, void *), void *func_arg2) +void hash_foreach(Hash* h, HFOREACH_FUN func, void *func_arg2) { int i; - for (i = 0; i < h->size; i++) { + for (i = 0; i < hash_get_slots(h); i++) { HashBucket* b = h->bucket[i]; while(b != (HashBucket*) 0) { (*func)((void *) b, func_arg2); diff --git a/erts/emulator/beam/hash.h b/erts/emulator/beam/hash.h index d319aaca83..4e8eb6594b 100644 --- a/erts/emulator/beam/hash.h +++ b/erts/emulator/beam/hash.h @@ -18,16 +18,16 @@ * %CopyrightEnd% */ -/* -** General hash functions -** -*/ +/** + * General hash functions + * + **/ #ifndef __HASH_H__ #define __HASH_H__ #include "sys.h" -typedef unsigned long HashValue; +typedef UWord HashValue; typedef struct hash Hash; typedef int (*HCMP_FUN)(void*, void*); @@ -38,6 +38,7 @@ typedef void (*HFREE_FUN)(void*); typedef void* (*HMALLOC_FUN)(int,size_t); typedef void (*HMFREE_FUN)(int,void*); typedef int (*HMPRINT_FUN)(fmtfn_t,void*,char*, ...); +typedef void (*HFOREACH_FUN)(void *, void *); /* ** This bucket must be placed in top of @@ -75,11 +76,10 @@ struct hash int is_allocated; /* 0 iff hash structure is on stack or is static */ int meta_alloc_type; /* argument to pass to meta_alloc and meta_free */ char* name; /* Table name (static string, for debugging) */ - int size; /* Number of slots */ + int shift; /* How much to shift the hash value */ + int max_shift; /* Never shift more than this value */ int shrink_threshold; int grow_threshold; - int size_ix; /* Size index in size table */ - int min_size_ix; /* Never shrink table smaller than this */ int nobjs; /* Number of objects in table */ HashBucket** bucket; /* Vector of bucket pointers (objects) */ }; @@ -96,6 +96,54 @@ void* hash_get(Hash*, void*); void* hash_put(Hash*, void*); void* hash_erase(Hash*, void*); void* hash_remove(Hash*, void*); -void hash_foreach(Hash*, void (*func)(void *, void *), void *); +void hash_foreach(Hash*, HFOREACH_FUN, void *); + +ERTS_GLB_INLINE Uint hash_get_slot(Hash *h, HashValue hv); +ERTS_GLB_INLINE void* hash_fetch(Hash *, void*, H_FUN, HCMP_FUN); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE Uint +hash_get_slot(Hash *h, HashValue hv) +{ + /* This slot mapping function uses fibonacci hashing in order to + * protect itself against a very bad hash function. This is not + * a hash function, so the user of hash.h should still spend time + * to figure out a good hash function for its data. + * + * See https://probablydance.com/2018/06/16/fibonacci-hashing-the-optimization-that-the-world-forgot-or-a-better-alternative-to-integer-modulo/ + * for some thoughts and ideas about fibonacci hashing. + */ + + /* This is not strictly part of the fibonacci hashing algorithm + * but it does help to spread the values of the mapping function better. + */ + hv ^= hv >> h->shift; +#ifdef ARCH_64 + /* 2^64 / 1.61803398875 = 11400714819323198485.... */ + return (UWORD_CONSTANT(11400714819323198485) * hv) >> h->shift; +#else + /* 2^32 / 1.61803398875 = 2654435769.... */ + return (UWORD_CONSTANT(2654435769) * hv) >> h->shift; +#endif +} + +ERTS_GLB_INLINE void* hash_fetch(Hash *h, void* tmpl, H_FUN hash, HCMP_FUN cmp) +{ + HashValue hval = hash(tmpl); + Uint ix = hash_get_slot(h, hval); + HashBucket* b = h->bucket[ix]; + ASSERT(h->fun.hash == hash); + ASSERT(h->fun.cmp == cmp); + + while(b != (HashBucket*) 0) { + if ((b->hvalue == hval) && (cmp(tmpl, (void*)b) == 0)) + return (void*) b; + b = b->next; + } + return (void*) 0; +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ #endif diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c index be1771b037..09d3c24424 100644 --- a/erts/emulator/beam/index.c +++ b/erts/emulator/beam/index.c @@ -114,35 +114,26 @@ int index_get(IndexTable* t, void* tmpl) return -1; } -void erts_index_merge(Hash* src, IndexTable* dst) +static void index_merge_foreach(IndexSlot *p, IndexTable *dst) { - int limit = src->size; - HashBucket** bucket = src->bucket; - int i; - - for (i = 0; i < limit; i++) { - HashBucket* b = bucket[i]; - IndexSlot* p; - int ix; - - while (b) { - Uint sz; - ix = dst->entries++; - if (ix >= dst->size) { - if (ix >= dst->limit) { - erts_exit(ERTS_ERROR_EXIT, "no more index entries in %s (max=%d)\n", - dst->htable.name, dst->limit); - } - sz = INDEX_PAGE_SIZE*sizeof(IndexSlot*); - dst->seg_table[ix>>INDEX_PAGE_SHIFT] = erts_alloc(dst->type, sz); - dst->size += INDEX_PAGE_SIZE; - } - p = (IndexSlot*) b; - p->index = ix; - dst->seg_table[ix>>INDEX_PAGE_SHIFT][ix&INDEX_PAGE_MASK] = p; - b = b->next; - } + Uint sz; + int ix = dst->entries++; + if (ix >= dst->size) { + if (ix >= dst->limit) { + erts_exit(ERTS_ERROR_EXIT, "no more index entries in %s (max=%d)\n", + dst->htable.name, dst->limit); + } + sz = INDEX_PAGE_SIZE*sizeof(IndexSlot*); + dst->seg_table[ix>>INDEX_PAGE_SHIFT] = erts_alloc(dst->type, sz); + dst->size += INDEX_PAGE_SIZE; } + p->index = ix; + dst->seg_table[ix>>INDEX_PAGE_SHIFT][ix&INDEX_PAGE_MASK] = p; +} + +void erts_index_merge(Hash* src, IndexTable* dst) +{ + hash_foreach(src, (HFOREACH_FUN)index_merge_foreach, dst); } void index_erase_latest_from(IndexTable* t, Uint from_ix) diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab index 7cffe7fb5c..f53d60a5db 100644 --- a/erts/emulator/beam/instrs.tab +++ b/erts/emulator/beam/instrs.tab @@ -19,7 +19,12 @@ // %CopyrightEnd% // -// Stack manipulation instructions +// +// Stack manipulation instructions follow. +// +// See the comment for AH() in macros.tab for information about +// the layout of stack frames. +// allocate(NeedStack, Live) { $AH($NeedStack, 0, $Live); @@ -58,105 +63,170 @@ allocate_heap_zero(NeedStack, NeedHeap, Live) { deallocate(Deallocate) { //| -no_prefetch - SET_CP(c_p, (BeamInstr *) cp_val(*E)); E = ADD_BYTE_OFFSET(E, $Deallocate); } -deallocate_return(Deallocate) { - //| -no_next - int words_to_pop = $Deallocate; - SET_I((BeamInstr *) cp_val(*E)); - E = ADD_BYTE_OFFSET(E, words_to_pop); - CHECK_TERM(x(0)); - DispatchReturn; +// +// Micro-benchmarks showed that the deallocate_return instruction +// became slower when the continuation pointer was moved from +// the process struct to the stack. The reason seems to be read +// dependencies, i.e. that the CPU cannot figure out beforehand +// from which position on the stack the continuation pointer +// should be fetched. +// +// Initializing num_bytes with a constant value seems to restore +// the lost speed, so we've specialized the instruction for the +// most common values. +// + +deallocate_return0 := dealloc_ret.n0.execute; +deallocate_return1 := dealloc_ret.n1.execute; +deallocate_return2 := dealloc_ret.n2.execute; +deallocate_return3 := dealloc_ret.n3.execute; +deallocate_return4 := dealloc_ret.n4.execute; +deallocate_return := dealloc_ret.var.execute; + +dealloc_ret.head() { + Uint num_bytes; } -move_deallocate_return(Src, Deallocate) { - x(0) = $Src; - $deallocate_return($Deallocate); +dealloc_ret.n0() { + num_bytes = (0+1) * sizeof(Eterm); } -// Call instructions +dealloc_ret.n1() { + num_bytes = (1+1) * sizeof(Eterm); +} + +dealloc_ret.n2() { + num_bytes = (2+1) * sizeof(Eterm); +} + +dealloc_ret.n3() { + num_bytes = (3+1) * sizeof(Eterm); +} + +dealloc_ret.n4() { + num_bytes = (4+1) * sizeof(Eterm); +} + +dealloc_ret.var(Deallocate) { + num_bytes = $Deallocate; +} -DISPATCH_REL(CallDest) { +dealloc_ret.execute() { //| -no_next - $SET_I_REL($CallDest); - DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); - Dispatch(); + + E = ADD_BYTE_OFFSET(E, num_bytes); + $RETURN(); + CHECK_TERM(x(0)); + $DISPATCH_RETURN(); } -DISPATCH_ABS(CallDest) { +move_deallocate_return(Src, Deallocate) { //| -no_next - SET_I((BeamInstr *) $CallDest); - DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); - Dispatch(); + + /* + * Explicitly do reads first to mitigate the impact of read + * dependencies. + */ + + Uint bytes_to_pop = $Deallocate; + Eterm src = $Src; + E = ADD_BYTE_OFFSET(E, bytes_to_pop); + x(0) = src; + DTRACE_RETURN_FROM_PC(c_p, I); + $RETURN(); + CHECK_TERM(x(0)); + $DISPATCH_RETURN(); } +// Call instructions + i_call(CallDest) { - SET_CP(c_p, $NEXT_INSTRUCTION); + //| -no_next + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); $DISPATCH_REL($CallDest); } move_call(Src, CallDest) { - x(0) = $Src; - SET_CP(c_p, $NEXT_INSTRUCTION); - $DISPATCH_REL($CallDest); + //| -no_next + Eterm call_dest = $CallDest; + Eterm src = $Src; + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); + x(0) = src; + $DISPATCH_REL(call_dest); } i_call_last(CallDest, Deallocate) { + //| -no_next $deallocate($Deallocate); $DISPATCH_REL($CallDest); } move_call_last(Src, CallDest, Deallocate) { - x(0) = $Src; - $i_call_last($CallDest, $Deallocate); + //| -no_next + Eterm call_dest = $CallDest; + Eterm src = $Src; + $deallocate($Deallocate); + x(0) = src; + $DISPATCH_REL(call_dest); } i_call_only(CallDest) { + //| -no_next $DISPATCH_REL($CallDest); } move_call_only(Src, CallDest) { - x(0) = $Src; - $i_call_only($CallDest); -} - -DISPATCHX(Dest) { //| -no_next - DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, $Dest); - // Dispatchx assumes the Export* is in Arg(0) - I = (&$Dest) - 1; - Dispatchx(); + Eterm call_dest = $CallDest; + Eterm src = $Src; + x(0) = src; + $DISPATCH_REL(call_dest); } i_call_ext(Dest) { - SET_CP(c_p, $NEXT_INSTRUCTION); - $DISPATCHX($Dest); + //| -no_next + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); + $DISPATCH_EXPORT($Dest); } -i_move_call_ext(Src, Dest) { - x(0) = $Src; - $i_call_ext($Dest); +i_move_call_ext(Src, CallDest) { + //| -no_next + Eterm call_dest = $CallDest; + Eterm src = $Src; + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); + x(0) = src; + $DISPATCH_EXPORT(call_dest); } i_call_ext_only(Dest) { - $DISPATCHX($Dest); + //| -no_next + $DISPATCH_EXPORT($Dest); } -i_move_call_ext_only(Dest, Src) { - x(0) = $Src; - $i_call_ext_only($Dest); +i_move_call_ext_only(CallDest, Src) { + //| -no_next + Eterm call_dest = $CallDest; + Eterm src = $Src; + x(0) = src; + $DISPATCH_EXPORT(call_dest); } i_call_ext_last(Dest, Deallocate) { + //| -no_next $deallocate($Deallocate); - $DISPATCHX($Dest); + $DISPATCH_EXPORT($Dest); } -i_move_call_ext_last(Dest, StackOffset, Src) { - x(0) = $Src; - $i_call_ext_last($Dest, $StackOffset); +i_move_call_ext_last(CallDest, Deallocate, Src) { + //| -no_next + Eterm call_dest = $CallDest; + Eterm src = $Src; + $deallocate($Deallocate); + x(0) = src; + $DISPATCH_EXPORT(call_dest); } APPLY(I, Deallocate, Next) { @@ -172,16 +242,18 @@ HANDLE_APPLY_ERROR() { } i_apply() { + //| -no_next BeamInstr *next; $APPLY(NULL, 0, next); if (ERTS_LIKELY(next != NULL)) { - SET_CP(c_p, $NEXT_INSTRUCTION); + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); $DISPATCH_ABS(next); } $HANDLE_APPLY_ERROR(); } i_apply_last(Deallocate) { + //| -no_next BeamInstr *next; $APPLY(I, $Deallocate, next); if (ERTS_LIKELY(next != NULL)) { @@ -192,6 +264,7 @@ i_apply_last(Deallocate) { } i_apply_only() { + //| -no_next BeamInstr *next; $APPLY(I, 0, next); if (ERTS_LIKELY(next != NULL)) { @@ -208,16 +281,18 @@ FIXED_APPLY(Arity, I, Deallocate, Next) { } apply(Arity) { + //| -no_next BeamInstr *next; $FIXED_APPLY($Arity, NULL, 0, next); if (ERTS_LIKELY(next != NULL)) { - SET_CP(c_p, $NEXT_INSTRUCTION); + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); $DISPATCH_ABS(next); } $HANDLE_APPLY_ERROR(); } apply_last(Arity, Deallocate) { + //| -no_next BeamInstr *next; $FIXED_APPLY($Arity, I, $Deallocate, next); if (ERTS_LIKELY(next != NULL)) { @@ -237,23 +312,19 @@ HANDLE_APPLY_FUN_ERROR() { goto find_func_info; } -DISPATCH_FUN(I) { - //| -no_next - SET_I($I); - Dispatchfun(); -} - i_apply_fun() { + //| -no_next BeamInstr *next; $APPLY_FUN(next); if (ERTS_LIKELY(next != NULL)) { - SET_CP(c_p, $NEXT_INSTRUCTION); + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); $DISPATCH_FUN(next); } $HANDLE_APPLY_FUN_ERROR(); } i_apply_fun_last(Deallocate) { + //| -no_next BeamInstr *next; $APPLY_FUN(next); if (ERTS_LIKELY(next != NULL)) { @@ -264,6 +335,7 @@ i_apply_fun_last(Deallocate) { } i_apply_fun_only() { + //| -no_next BeamInstr *next; $APPLY_FUN(next); if (ERTS_LIKELY(next != NULL)) { @@ -280,16 +352,18 @@ CALL_FUN(Fun, Next) { } i_call_fun(Fun) { + //| -no_next BeamInstr *next; $CALL_FUN($Fun, next); if (ERTS_LIKELY(next != NULL)) { - SET_CP(c_p, $NEXT_INSTRUCTION); + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); $DISPATCH_FUN(next); } $HANDLE_APPLY_FUN_ERROR(); } i_call_fun_last(Fun, Deallocate) { + //| -no_next BeamInstr *next; $CALL_FUN($Fun, next); if (ERTS_LIKELY(next != NULL)) { @@ -301,18 +375,12 @@ i_call_fun_last(Fun, Deallocate) { return() { //| -no_next - SET_I(c_p->cp); - DTRACE_RETURN_FROM_PC(c_p); - - /* - * We must clear the CP to make sure that a stale value do not - * create a false module dependcy preventing code upgrading. - * It also means that we can use the CP in stack backtraces. - */ - c_p->cp = 0; + DTRACE_RETURN_FROM_PC(c_p, I); + $RETURN(); CHECK_TERM(r(0)); HEAP_SPACE_VERIFIED(0); - DispatchReturn; + + $DISPATCH_RETURN(); } get_list(Src, Hd, Tl) { @@ -478,16 +546,21 @@ i_make_fun(FunP, NumFree) { } move_trim(Src, Dst, Words) { - Uint cp = E[0]; $Dst = $Src; - E += $Words; - E[0] = cp; + $i_trim($Words); } i_trim(Words) { - Uint cp = E[0]; E += $Words; - E[0] = cp; + + /* + * Clear the reserved location for the continuation pointer at + * E[0]. This is not strictly necessary for correctness, but if a + * GC is triggered before E[0] is overwritten by another + * continuation pointer the now dead term at E[0] would be + * retained by the GC. + */ + E[0] = NIL; } move(Src, Dst) { @@ -599,9 +672,9 @@ move_window5(S1, S2, S3, S4, S5, D) { move_return(Src) { //| -no_next x(0) = $Src; - SET_I(c_p->cp); - c_p->cp = 0; - DispatchReturn; + DTRACE_RETURN_FROM_PC(c_p, I); + $RETURN(); + $DISPATCH_RETURN(); } move_x1(Src) { @@ -683,10 +756,11 @@ swap(R1, R2) { $R2 = V; } -swap_temp(R1, R2, Tmp) { - Eterm V = $R1; - $R1 = $R2; - $R2 = $Tmp = V; +swap2(R1, R2, R3) { + Eterm V = $R2; + $R2 = $R1; + $R1 = $R3; + $R3 = V; } test_heap(Nh, Live) { diff --git a/erts/emulator/beam/macros.tab b/erts/emulator/beam/macros.tab index 1b5e5f66b0..848e35d45c 100644 --- a/erts/emulator/beam/macros.tab +++ b/erts/emulator/beam/macros.tab @@ -104,14 +104,136 @@ GC_TEST_PRESERVE(NeedHeap, Live, PreserveTerm) { // Make sure that there are NeedStack + NeedHeap + 1 words available -// on the combined heap/stack segment, then allocates NeedHeap + 1 -// words on the stack and saves CP. +// on the combined heap/stack segment, then decrement the stack +// pointer by (NeedStack + 1) words. Finally clear the word reserved +// for the continuation pointer at the top of the stack. +// +// Stack frame layout: +// +// +-----------+ +// y(N) | Term | +// +-----------+ +// . +// . +// . +// +-----------+ +// y(0) | Term | +// +-----------+ +// E ==> | NIL or CP | +// +-----------+ +// +// When the function owning the stack frame is the currently executing +// function, the word at the top of the stack is NIL. When calling +// another function, the continuation pointer will be stored in the +// word at the top of the stack. When returning to the function +// owning the stack frame, the word at the stack top will again be set +// to NIL. + AH(NeedStack, NeedHeap, Live) { unsigned needed = $NeedStack + 1; $GC_TEST(needed, $NeedHeap, $Live); E -= needed; - *E = make_cp(c_p->cp); - c_p->cp = 0; + *E = NIL; +} + + +// +// Helpers for call instructions +// + +DISPATCH() { + BeamInstr dis_next; + + dis_next = *I; + CHECK_ARGS(I); + + if (FCALLS > 0 || FCALLS > neg_o_reds) { + FCALLS--; + Goto(dis_next); + } else { + goto context_switch; + } +} + +DISPATCH_ABS(CallDest) { + SET_I((BeamInstr *) $CallDest); + DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); + + $DISPATCH(); +} + +DISPATCH_EXPORT(Export) { + BeamInstr dis_next; + Export *ep; + + ep = (Export*)($Export); + + DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, ep); + + SET_I(ep->addressv[erts_active_code_ix()]); + CHECK_ARGS(I); + dis_next = *I; + + if (ERTS_UNLIKELY(FCALLS <= 0)) { + if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p) && FCALLS > neg_o_reds) { + save_calls(c_p, ep); + } else { + goto context_switch; + } + } + + FCALLS--; + Goto(dis_next); +} + +DISPATCH_FUN(I) { + BeamInstr dis_next; + + SET_I($I); + + dis_next = *I; + CHECK_ARGS(I); + + if (FCALLS > 0 || FCALLS > neg_o_reds) { + FCALLS--; + Goto(dis_next); + } else { + goto context_switch_fun; + } +} + +DISPATCH_REL(CallDest) { + $SET_I_REL($CallDest); + DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); + + $DISPATCH(); +} + +DISPATCH_RETURN() { + if (FCALLS > 0 || FCALLS > neg_o_reds) { + FCALLS--; + Goto(*I); + } else { + c_p->current = NULL; + c_p->arity = 1; + goto context_switch3; + } +} + +// Save the continuation pointer in the reserved slot at the +// top of the stack as preparation for doing a function call. + +SAVE_CONTINUATION_POINTER(IP) { + ASSERT(VALID_INSTR(*($IP))); + *E = (BeamInstr) ($IP); +} + +// Return to the function whose continuation pointer is stored +// at the top of the stack and set that word to NIL. + +RETURN() { + SET_I(cp_val(*E)); + *E = NIL; } NEXT0() { diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index b9d4f6afcc..1d336e4b7b 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -77,19 +77,32 @@ return # To ensure that a "move Src x(0)" instruction can be combined with # the following call instruction, we need to make sure that there is # no line/1 instruction between the move and the call. -# -# A tail-recursive call to an external function (BIF or non-BIF) will -# never be saved on the stack, so there is no reason to keep the line -# instruction. + +move S X0=x==0 | line Loc | call Ar Func => \ + line Loc | move S X0 | call Ar Func move S X0=x==0 | line Loc | call_ext Ar Func => \ line Loc | move S X0 | call_ext Ar Func + +# +# A tail call will not refer to the current function on error unless it's a +# BIF, so we can omit the line instruction for non-BIFs. +# + +move S X0=x==0 | line Loc | call_ext_last Ar Func=u$is_bif D => \ + line Loc | move S X0 | call_ext_last Ar Func D +move S X0=x==0 | line Loc | call_ext_only Ar Func=u$is_bif => \ + line Loc | move S X0 | call_ext_only Ar Func + move S X0=x==0 | line Loc | call_ext_last Ar Func D => \ move S X0 | call_ext_last Ar Func D move S X0=x==0 | line Loc | call_ext_only Ar Func => \ move S X0 | call_ext_only Ar Func -move S X0=x==0 | line Loc | call Ar Func => \ - line Loc | move S X0 | call Ar Func + +move S X0=x==0 | line Loc | call_last Ar Func D => \ + move S X0 | call_last Ar Func D +move S X0=x==0 | line Loc | call_only Ar Func => \ + move S X0 | call_only Ar Func line Loc | func_info M F A => func_info M F A | line Loc @@ -324,76 +337,15 @@ move_src_window2 y x x move_src_window3 y x x x move_src_window4 y x x x x -# Swap registers. -move R1=xy Tmp=x | move R2=xy R1 | move Tmp R2 => swap_temp R1 R2 Tmp - -# The compiler uses x(1022) when swapping registers. It will definitely -# not be used again. -swap_temp R1 R2 Tmp=x==1022 => swap R1 R2 - -swap_temp R1 R2 Tmp | move Src Tmp => swap R1 R2 | move Src Tmp - -swap_temp R1 R2 Tmp | line Loc | apply Live | is_killed_apply(Tmp, Live) => \ - swap R1 R2 | line Loc | apply Live -swap_temp R1 R2 Tmp | line Loc | apply_last Live D | is_killed_apply(Tmp, Live) => \ - swap R1 R2 | line Loc | apply_last Live D - -swap_temp R1 R2 Tmp | line Loc | call_fun Live | is_killed_by_call_fun(Tmp, Live) => \ - swap R1 R2 | line Loc | call_fun Live -swap_temp R1 R2 Tmp | make_fun2 OldIndex=u | is_killed_by_make_fun(Tmp, OldIndex) => \ - swap R1 R2 | make_fun2 OldIndex - -swap_temp R1 R2 Tmp | line Loc | call Live Addr | is_killed(Tmp, Live) => \ - swap R1 R2 | line Loc | call Live Addr -swap_temp R1 R2 Tmp | call_only Live Addr | \ - is_killed(Tmp, Live) => swap R1 R2 | call_only Live Addr -swap_temp R1 R2 Tmp | call_last Live Addr D | \ - is_killed(Tmp, Live) => swap R1 R2 | call_last Live Addr D - -swap_temp R1 R2 Tmp | line Loc | call_ext Live Addr | is_killed(Tmp, Live) => \ - swap R1 R2 | line Loc | call_ext Live Addr -swap_temp R1 R2 Tmp | line Loc | call_ext_only Live Addr | \ - is_killed(Tmp, Live) => swap R1 R2 | line Loc | call_ext_only Live Addr -swap_temp R1 R2 Tmp | line Loc | call_ext_last Live Addr D | \ - is_killed(Tmp, Live) => swap R1 R2 | line Loc | call_ext_last Live Addr D - -swap_temp R1 R2 Tmp | call_ext Live Addr | is_killed(Tmp, Live) => \ - swap R1 R2 | call_ext Live Addr -swap_temp R1 R2 Tmp | call_ext_only Live Addr | is_killed(Tmp, Live) => \ - swap R1 R2 | call_ext_only Live Addr -swap_temp R1 R2 Tmp | call_ext_last Live Addr D | is_killed(Tmp, Live) => \ - swap R1 R2 | call_ext_last Live Addr D - -swap_temp R1 R2 Tmp | move Src Any | line Loc | call Live Addr | \ - is_killed(Tmp, Live) | distinct(Tmp, Src) => \ - swap R1 R2 | move Src Any | line Loc | call Live Addr -swap_temp R1 R2 Tmp | move Src Any | line Loc | call_ext Live Addr | \ - is_killed(Tmp, Live) | distinct(Tmp, Src) => \ - swap R1 R2 | move Src Any | line Loc | call_ext Live Addr -swap_temp R1 R2 Tmp | move Src Any | call_only Live Addr | \ - is_killed(Tmp, Live) | distinct(Tmp, Src) => \ - swap R1 R2 | move Src Any | call_only Live Addr -swap_temp R1 R2 Tmp | move Src Any | line Loc | call_ext_only Live Addr | \ - is_killed(Tmp, Live) | distinct(Tmp, Src) => \ - swap R1 R2 | move Src Any | line Loc | call_ext_only Live Addr -swap_temp R1 R2 Tmp | move Src Any | line Loc | call_fun Live | \ - is_killed(Tmp, Live) | distinct(Tmp, Src) => \ - swap R1 R2 | move Src Any | line Loc | call_fun Live - -swap_temp R1 R2 Tmp | line Loc | send | is_killed_by_send(Tmp) => \ - swap R1 R2 | line Loc | send - -# swap_temp/3 with Y register operands are rare. -swap_temp R1 R2=y Tmp => swap R1 R2 | move R2 Tmp -swap_temp R1=y R2 Tmp => swap R1 R2 | move R2 Tmp - swap R1=x R2=y => swap R2 R1 -swap_temp x x x - swap xy x swap y y +swap R1=x R2=x | swap R3=x R1 => swap2 R1 R2 R3 + +swap2 x x x + # move_shift move SD=x D=x | move Src=cxy SD=x | distinct(D, Src) => move_shift Src SD D @@ -635,8 +587,8 @@ put_list s s d %cold normal_exit continue_exit -apply_bif -call_nif +call_bif W +call_nif W W W call_error_handler error_action_code return_trace @@ -657,8 +609,20 @@ move S x==0 | deallocate D | return => move_deallocate_return S D move_deallocate_return xycn Q +deallocate u==0 | return => deallocate_return0 +deallocate u==1 | return => deallocate_return1 +deallocate u==2 | return => deallocate_return2 +deallocate u==3 | return => deallocate_return3 +deallocate u==4 | return => deallocate_return4 + deallocate D | return => deallocate_return D +deallocate_return0 +deallocate_return1 +deallocate_return2 +deallocate_return3 +deallocate_return4 + deallocate_return Q test_heap Need u==1 | put_list Y=y x==0 x==0 => test_heap_1_put_list Need Y @@ -836,62 +800,22 @@ allocate_init t t? y # External function and bif calls. ################################################################# -# -# The BIFs erts_internal:check_process_code/1 must be called like a function, -# to ensure that c_p->i (program counter) is set correctly (an ordinary -# BIF call doesn't set it). -# - -call_ext u==1 Bif=u$bif:erts_internal:check_process_code/1 => i_call_ext Bif -call_ext_last u==1 Bif=u$bif:erts_internal:check_process_code/1 D => i_call_ext_last Bif D -call_ext_only u==1 Bif=u$bif:erts_internal:check_process_code/1 => i_call_ext_only Bif +# Expands into call_light_bif(_only)/2 +call_light_bif/1 +call_light_bif_only/1 +call_light_bif_last/2 # -# The BIFs erts_internal:garbage_collect/1 must be called like a function, -# to allow them to invoke the garbage collector. (The stack pointer must -# be saved and p->arity must be zeroed, which is not done on ordinary BIF calls.) -# -call_ext u==1 Bif=u$bif:erts_internal:garbage_collect/1 => i_call_ext Bif -call_ext_last u==1 Bif=u$bif:erts_internal:garbage_collect/1 D => i_call_ext_last Bif D -call_ext_only u==1 Bif=u$bif:erts_internal:garbage_collect/1 => i_call_ext_only Bif - -# -# put/2 and erase/1 must be able to do garbage collection, so we must call -# them like functions. -# - -call_ext u==2 Bif=u$bif:erlang:put/2 => i_call_ext Bif -call_ext_last u==2 Bif=u$bif:erlang:put/2 D => i_call_ext_last Bif D -call_ext_only u==2 Bif=u$bif:erlang:put/2 => i_call_ext_only Bif - -call_ext u==1 Bif=u$bif:erlang:erase/1 => i_call_ext Bif -call_ext_last u==1 Bif=u$bif:erlang:erase/1 D => i_call_ext_last Bif D -call_ext_only u==1 Bif=u$bif:erlang:erase/1 => i_call_ext_only Bif - -# -# The process_info/1,2 BIF should be called like a function, to force -# the emulator to set c_p->current before calling it (a BIF call doesn't -# set it). -# -# In addition, we force the use of a non-tail-recursive call. This will ensure -# that c_p->cp points into the function making the call. +# The load_nif/2 BIF is an instruction. # -call_ext u==1 Bif=u$bif:erlang:process_info/1 => i_call_ext Bif -call_ext_last u==1 Bif=u$bif:erlang:process_info/1 D => i_call_ext Bif | deallocate_return D -call_ext_only Ar=u==1 Bif=u$bif:erlang:process_info/1 => allocate u Ar | i_call_ext Bif | deallocate_return u - -call_ext u==2 Bif=u$bif:erlang:process_info/2 => i_call_ext Bif -call_ext_last u==2 Bif=u$bif:erlang:process_info/2 D => i_call_ext Bif | deallocate_return D -call_ext_only Ar=u==2 Bif=u$bif:erlang:process_info/2 => allocate u Ar | i_call_ext Bif | deallocate_return u - -# -# load_nif/2 also needs to know calling function like process_info -# -call_ext u==2 Bif=u$bif:erlang:load_nif/2 => i_call_ext Bif -call_ext_last u==2 Bif=u$bif:erlang:load_nif/2 D => i_call_ext Bif | deallocate_return D -call_ext_only Ar=u==2 Bif=u$bif:erlang:load_nif/2 => allocate u Ar | i_call_ext Bif | deallocate_return u +call_ext u==2 u$func:erlang:load_nif/2 => i_load_nif +call_ext_last u==2 u$func:erlang:load_nif/2 D => i_load_nif | deallocate_return D +call_ext_only u==2 u$func:erlang:load_nif/2 => i_load_nif | return +%cold +i_load_nif +%hot # # apply/2 is an instruction, not a BIF. @@ -910,33 +834,6 @@ call_ext_last u==3 u$bif:erlang:apply/3 D => i_apply_last D call_ext_only u==3 u$bif:erlang:apply/3 => i_apply_only # -# The exit/1 and throw/1 BIFs never execute the instruction following them; -# thus there is no need to generate any return instruction. -# - -call_ext_last u==1 Bif=u$bif:erlang:exit/1 D => call_bif Bif -call_ext_last u==1 Bif=u$bif:erlang:throw/1 D => call_bif Bif - -call_ext_only u==1 Bif=u$bif:erlang:exit/1 => call_bif Bif -call_ext_only u==1 Bif=u$bif:erlang:throw/1 => call_bif Bif - -# -# The error/1 and error/2 BIFs never execute the instruction following them; -# thus there is no need to generate any return instruction. -# However, they generate stack backtraces, so if the call instruction -# is call_ext_only/2 instruction, we explicitly do an allocate/2 to store -# the continuation pointer on the stack. -# - -call_ext_last u==1 Bif=u$bif:erlang:error/1 D => call_bif Bif -call_ext_last u==2 Bif=u$bif:erlang:error/2 D => call_bif Bif - -call_ext_only Ar=u==1 Bif=u$bif:erlang:error/1 => \ - allocate u Ar | call_bif Bif -call_ext_only Ar=u==2 Bif=u$bif:erlang:error/2 => \ - allocate u Ar | call_bif Bif - -# # The yield/0 BIF is an instruction # @@ -1048,17 +945,24 @@ call_ext_only u==0 u$func:os:perf_counter/0 => \ i_perf_counter | return # -# The general case for BIFs that have no special instructions. -# A BIF used in the tail must be followed by a return instruction. +# BIFs like process_info/1,2 require up-to-date information about the current +# emulator state, which the ordinary call_light_bif instruction doesn't save. # -# To make trapping and stack backtraces work correctly, we make sure that -# the continuation pointer is always stored on the stack. -call_ext u Bif=u$is_bif => call_bif Bif +call_ext u Bif=u$is_bif | is_heavy_bif(Bif) => \ + i_call_ext Bif +call_ext_last u Bif=u$is_bif D | is_heavy_bif(Bif) => \ + i_call_ext Bif | deallocate_return D +call_ext_only Ar=u Bif=u$is_bif | is_heavy_bif(Bif) => \ + allocate u Ar | i_call_ext Bif | deallocate_return u -call_ext_last u Bif=u$is_bif D => deallocate D | call_bif_only Bif +# +# The general case for BIFs that have no special requirements. +# -call_ext_only Ar=u Bif=u$is_bif => call_bif_only Bif +call_ext u Bif=u$is_bif => call_light_bif Bif +call_ext_last u Bif=u$is_bif D => call_light_bif_last Bif D +call_ext_only Ar=u Bif=u$is_bif => call_light_bif_only Bif # # Any remaining calls are calls to Erlang functions, not BIFs. @@ -1083,14 +987,32 @@ i_apply_fun i_apply_fun_last Q i_apply_fun_only +# +# When a BIF is traced, these instructions make a body call through the export +# entry instead of calling the BIF directly (setting up a temporary stack frame +# if needed). We therefore retain the stack frame in call_light_bif_last, and +# add a deallocate_return after call_light_bif_only to remove the temporary +# stack frame before returning. +# + +call_light_bif Bif=u$is_bif => \ + call_light_bif Bif Bif + +call_light_bif_last Bif=u$is_bif D => \ + call_light_bif Bif Bif | deallocate_return D + +call_light_bif_only Bif=u$is_bif => \ + call_light_bif_only Bif Bif | deallocate_return u + +call_light_bif b e +call_light_bif_only b e + %cold -i_hibernate +i_hibernate i_perf_counter -%hot -call_bif e -call_bif_only e +%hot # # Calls to non-building and guard BIFs. diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c index c7e02c6d48..8e44b527a2 100644 --- a/erts/emulator/beam/register.c +++ b/erts/emulator/beam/register.c @@ -265,10 +265,8 @@ Eterm erts_whereis_name_to_id(Process *c_p, Eterm name) { Eterm res = am_undefined; - HashValue hval; - int ix; - HashBucket* b; ErtsProcLocks c_p_locks = 0; + RegProc *rp, tmpl; if (c_p) { c_p_locks = ERTS_PROC_LOCK_MAIN; ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(c_p); @@ -278,29 +276,14 @@ erts_whereis_name_to_id(Process *c_p, Eterm name) if (c_p && !c_p_locks) erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); - hval = REG_HASH(name); - ix = hval % process_reg.size; - b = process_reg.bucket[ix]; + tmpl.name = name; + rp = hash_fetch(&process_reg, &tmpl, (H_FUN)reg_hash, (HCMP_FUN)reg_cmp); - /* - * Note: We have inlined the code from hash.c for speed. - */ - - while (b) { - RegProc* rp = (RegProc *) b; - if (rp->name == name) { - /* - * SMP NOTE: No need to lock registered entity since it cannot - * be removed without acquiring write reg lock and id on entity - * is read only. - */ - if (rp->p) - res = rp->p->common.id; - else if (rp->pt) - res = rp->pt->common.id; - break; - } - b = b->next; + if (rp) { + if (rp->p) + res = rp->p->common.id; + else if (rp->pt) + res = rp->pt->common.id; } reg_read_unlock(); @@ -321,10 +304,7 @@ erts_whereis_name(Process *c_p, Port** port, int lock_port) { - RegProc* rp = NULL; - HashValue hval; - int ix; - HashBucket* b; + RegProc* rp = NULL, tmpl; ErtsProcLocks current_c_p_locks; Port *pending_port = NULL; @@ -342,21 +322,8 @@ erts_whereis_name(Process *c_p, * - current_c_p_locks (either c_p_locks or 0) on c_p */ - hval = REG_HASH(name); - ix = hval % process_reg.size; - b = process_reg.bucket[ix]; - - /* - * Note: We have inlined the code from hash.c for speed. - */ - - while (b) { - if (((RegProc *) b)->name == name) { - rp = (RegProc *) b; - break; - } - b = b->next; - } + tmpl.name = name; + rp = hash_fetch(&process_reg, &tmpl, (H_FUN)reg_hash, (HCMP_FUN)reg_cmp); if (proc) { if (!rp) @@ -564,18 +531,6 @@ int erts_unregister_name(Process *c_p, return res; } -int process_reg_size(void) -{ - int size; - int lock = !ERTS_IS_CRASH_DUMPING; - if (lock) - reg_read_lock(); - size = process_reg.size; - if (lock) - reg_read_unlock(); - return size; -} - int process_reg_sz(void) { int sz; @@ -592,15 +547,24 @@ int process_reg_sz(void) #include "bif.h" +struct registered_foreach_arg { + Eterm res; + Eterm *hp; +}; + +static void +registered_foreach(RegProc *reg, struct registered_foreach_arg *arg) +{ + arg->res = CONS(arg->hp, reg->name, arg->res); + arg->hp += 2; +} + /* return a list of the registered processes */ BIF_RETTYPE registered_0(BIF_ALIST_0) { - int i; - Eterm res; + struct registered_foreach_arg arg; Uint need; - Eterm* hp; - HashBucket **bucket; ErtsProcLocks proc_locks = ERTS_PROC_LOCK_MAIN; ERTS_CHK_HAVE_ONLY_MAIN_PROC_LOCK(BIF_P); @@ -608,41 +572,21 @@ BIF_RETTYPE registered_0(BIF_ALIST_0) if (!proc_locks) erts_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); - bucket = process_reg.bucket; - - /* work out how much heap we need & maybe garb, by scanning through - the registered process table */ - need = 0; - for (i = 0; i < process_reg.size; i++) { - HashBucket *b = bucket[i]; - while (b != NULL) { - need += 2; - b = b->next; - } - } + /* work out how much heap we need */ + need = process_reg.nobjs * 2; if (need == 0) { reg_read_unlock(); BIF_RET(NIL); } - hp = HAlloc(BIF_P, need); - - /* scan through again and make the list */ - res = NIL; + /* scan through again and make the list */ + arg.hp = HAlloc(BIF_P, need); + arg.res = NIL; - for (i = 0; i < process_reg.size; i++) { - HashBucket *b = bucket[i]; - while (b != NULL) { - RegProc *reg = (RegProc *) b; - - res = CONS(hp, reg->name, res); - hp += 2; - b = b->next; - } - } + hash_foreach(&process_reg, (HFOREACH_FUN)registered_foreach, &arg); reg_read_unlock(); - BIF_RET(res); + BIF_RET(arg.res); } diff --git a/erts/emulator/beam/register.h b/erts/emulator/beam/register.h index 27a314ca78..c77bd03653 100644 --- a/erts/emulator/beam/register.h +++ b/erts/emulator/beam/register.h @@ -41,7 +41,6 @@ typedef struct reg_proc Eterm name; /* Atom name */ } RegProc; -int process_reg_size(void); int process_reg_sz(void); void init_register_table(void); void register_info(fmtfn_t, void *); diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index c261c8e117..8a59b61b63 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -92,6 +92,12 @@ # define ERTS_GLB_INLINE_INCL_FUNC_DEF 0 #endif +#ifdef __GNUC__ +# define ERTS_NOINLINE __attribute__((__noinline__)) +#else +# define ERTS_NOINLINE +#endif + #if defined(VALGRIND) && !defined(NO_FPE_SIGNALS) # define NO_FPE_SIGNALS #endif @@ -172,7 +178,8 @@ typedef ERTS_SYS_FD_TYPE ErtsSysFdType; # define ERTS_UNLIKELY(BOOL) (BOOL) #endif -#if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0) +/* AIX doesn't like this and claims section conflicts */ +#if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0) && !defined(_AIX) #if (defined(__APPLE__) && defined(__MACH__)) || defined(__DARWIN__) # define ERTS_WRITE_UNLIKELY(X) X __attribute__ ((section ("__DATA,ERTS_LOW_WRITE") )) #else @@ -666,7 +673,16 @@ typedef struct preload { */ typedef Eterm ErtsTracer; -#include "erl_osenv.h" + +/* + * This structure contains the rb tree for the erlang osenv copy + * see erl_osenv.h for more details. + */ +typedef struct __erts_osenv_t { + struct __env_rbtnode_t *tree; + int variable_count; + int content_size; +} erts_osenv_t; /* * This structure contains options to all built in drivers. diff --git a/erts/emulator/beam/trace_instrs.tab b/erts/emulator/beam/trace_instrs.tab index 3eee81c053..9f22587f96 100644 --- a/erts/emulator/beam/trace_instrs.tab +++ b/erts/emulator/beam/trace_instrs.tab @@ -20,16 +20,15 @@ // return_trace() { - ErtsCodeMFA* mfa = (ErtsCodeMFA *)(E[0]); + ErtsCodeMFA* mfa = (ErtsCodeMFA *)(E[1]); SWAPOUT; /* Needed for shared heap */ ERTS_UNREQ_PROC_MAIN_LOCK(c_p); - erts_trace_return(c_p, mfa, r(0), ERTS_TRACER_FROM_ETERM(E+1)/* tracer */); + erts_trace_return(c_p, mfa, r(0), ERTS_TRACER_FROM_ETERM(E+2)/* tracer */); ERTS_REQ_PROC_MAIN_LOCK(c_p); SWAPIN; - c_p->cp = NULL; - SET_I((BeamInstr *) cp_val(E[2])); E += 3; + $RETURN(); Goto(*I); //| -no_next } @@ -45,13 +44,12 @@ i_generic_breakpoint() { } i_return_time_trace() { - BeamInstr *pc = (BeamInstr *) (UWord) E[0]; + BeamInstr *pc = (BeamInstr *) (UWord) E[1]; SWAPOUT; erts_trace_time_return(c_p, erts_code_to_codeinfo(pc)); SWAPIN; - c_p->cp = NULL; - SET_I((BeamInstr *) cp_val(E[1])); E += 2; + $RETURN(); Goto(*I); //| -no_next } @@ -59,8 +57,10 @@ i_return_time_trace() { i_return_to_trace() { if (IS_TRACED_FL(c_p, F_TRACE_RETURN_TO)) { Uint *cpp = (Uint*) E; + while (is_not_CP(*cpp)) { + cpp++; + } for(;;) { - ASSERT(is_CP(*cpp)); if (IsOpCode(*cp_val(*cpp), return_trace)) { do ++cpp; @@ -80,9 +80,8 @@ i_return_to_trace() { ERTS_REQ_PROC_MAIN_LOCK(c_p); SWAPIN; } - c_p->cp = NULL; - SET_I((BeamInstr *) cp_val(E[0])); E += 1; + $RETURN(); Goto(*I); //| -no_next } diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 0bbae65e28..fb06d60768 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -66,7 +66,7 @@ #undef M_MMAP_THRESHOLD #undef M_MMAP_MAX -#if defined(__GLIBC__) && defined(HAVE_MALLOC_H) +#if (defined(__GLIBC__) || defined(_AIX)) && defined(HAVE_MALLOC_H) #include <malloc.h> #endif @@ -907,7 +907,7 @@ tail_recur: hash = hash * FUNNY_NUMBER10 + num_free; hash = hash*FUNNY_NUMBER1 + (atom_tab(atom_val(funp->fe->module))->slot.bucket.hvalue); - hash = hash*FUNNY_NUMBER2 + funp->fe->old_index; + hash = hash*FUNNY_NUMBER2 + funp->fe->index; hash = hash*FUNNY_NUMBER2 + funp->fe->old_uniq; if (num_free > 0) { if (num_free > 1) { @@ -1069,54 +1069,237 @@ do { \ #define HCONST 0x9e3779b9UL /* the golden ratio; an arbitrary value */ -static Uint32 -block_hash(byte *k, Uint length, Uint32 initval) +typedef struct { + Uint32 a,b,c; +} ErtsBlockHashHelperCtx; + +#define BLOCK_HASH_BYTES_PER_ITER 12 + +/* The three functions below are separated into different functions even + though they are always used together to make trapping and handling + of unaligned binaries easier. Examples of how they are used can be + found in block_hash and make_hash2_helper.*/ +static ERTS_INLINE +void block_hash_setup(Uint32 initval, + ErtsBlockHashHelperCtx* ctx /* out parameter */) +{ + ctx->a = ctx->b = HCONST; + ctx->c = initval; /* the previous hash value */ +} + +static ERTS_INLINE +void block_hash_buffer(byte *buf, + Uint buf_length, + ErtsBlockHashHelperCtx* ctx /* out parameter */) { - Uint32 a,b,c; - Uint len; - - /* Set up the internal state */ - len = length; - a = b = HCONST; - c = initval; /* the previous hash value */ - - while (len >= 12) - { - a += (k[0] +((Uint32)k[1]<<8) +((Uint32)k[2]<<16) +((Uint32)k[3]<<24)); - b += (k[4] +((Uint32)k[5]<<8) +((Uint32)k[6]<<16) +((Uint32)k[7]<<24)); - c += (k[8] +((Uint32)k[9]<<8) +((Uint32)k[10]<<16)+((Uint32)k[11]<<24)); - MIX(a,b,c); - k += 12; len -= 12; - } - - c += length; - switch(len) /* all the case statements fall through */ - { - case 11: c+=((Uint32)k[10]<<24); - case 10: c+=((Uint32)k[9]<<16); - case 9 : c+=((Uint32)k[8]<<8); - /* the first byte of c is reserved for the length */ - case 8 : b+=((Uint32)k[7]<<24); - case 7 : b+=((Uint32)k[6]<<16); - case 6 : b+=((Uint32)k[5]<<8); - case 5 : b+=k[4]; - case 4 : a+=((Uint32)k[3]<<24); - case 3 : a+=((Uint32)k[2]<<16); - case 2 : a+=((Uint32)k[1]<<8); - case 1 : a+=k[0]; - /* case 0: nothing left to add */ - } - MIX(a,b,c); - return c; + Uint len = buf_length; + byte *k = buf; + ASSERT(buf_length % BLOCK_HASH_BYTES_PER_ITER == 0); + while (len >= BLOCK_HASH_BYTES_PER_ITER) { + ctx->a += (k[0] +((Uint32)k[1]<<8) +((Uint32)k[2]<<16) +((Uint32)k[3]<<24)); + ctx->b += (k[4] +((Uint32)k[5]<<8) +((Uint32)k[6]<<16) +((Uint32)k[7]<<24)); + ctx->c += (k[8] +((Uint32)k[9]<<8) +((Uint32)k[10]<<16)+((Uint32)k[11]<<24)); + MIX(ctx->a,ctx->b,ctx->c); + k += BLOCK_HASH_BYTES_PER_ITER; len -= BLOCK_HASH_BYTES_PER_ITER; + } } +static ERTS_INLINE +Uint32 block_hash_final_bytes(byte *buf, + Uint buf_length, + Uint full_length, + ErtsBlockHashHelperCtx* ctx) +{ + Uint len = buf_length; + byte *k = buf; + ctx->c += full_length; + switch(len) + { /* all the case statements fall through */ + case 11: ctx->c+=((Uint32)k[10]<<24); + case 10: ctx->c+=((Uint32)k[9]<<16); + case 9 : ctx->c+=((Uint32)k[8]<<8); + /* the first byte of c is reserved for the length */ + case 8 : ctx->b+=((Uint32)k[7]<<24); + case 7 : ctx->b+=((Uint32)k[6]<<16); + case 6 : ctx->b+=((Uint32)k[5]<<8); + case 5 : ctx->b+=k[4]; + case 4 : ctx->a+=((Uint32)k[3]<<24); + case 3 : ctx->a+=((Uint32)k[2]<<16); + case 2 : ctx->a+=((Uint32)k[1]<<8); + case 1 : ctx->a+=k[0]; + /* case 0: nothing left to add */ + } + MIX(ctx->a,ctx->b,ctx->c); + return ctx->c; +} + +static Uint32 -make_hash2(Eterm term) +block_hash(byte *block, Uint block_length, Uint32 initval) { + ErtsBlockHashHelperCtx ctx; + Uint no_bytes_not_in_loop = + (block_length % BLOCK_HASH_BYTES_PER_ITER); + Uint no_bytes_to_process_in_loop = + block_length - no_bytes_not_in_loop; + byte *final_bytes = block + no_bytes_to_process_in_loop; + block_hash_setup(initval, &ctx); + block_hash_buffer(block, + no_bytes_to_process_in_loop, + &ctx); + return block_hash_final_bytes(final_bytes, + no_bytes_not_in_loop, + block_length, + &ctx); +} + +typedef enum { + tag_primary_list, + arityval_subtag, + hamt_subtag_head_flatmap, + map_subtag, + fun_subtag, + neg_big_subtag, + sub_binary_subtag_1, + sub_binary_subtag_2, + hash2_common_1, + hash2_common_2, + hash2_common_3, +} ErtsMakeHash2TrapLocation; + +typedef struct { + int c; + Uint32 sh; + Eterm* ptr; +} ErtsMakeHash2Context_TAG_PRIMARY_LIST; + +typedef struct { + int i; + int arity; + Eterm* elem; +} ErtsMakeHash2Context_ARITYVAL_SUBTAG; + +typedef struct { + Eterm *ks; + Eterm *vs; + int i; + Uint size; +} ErtsMakeHash2Context_HAMT_SUBTAG_HEAD_FLATMAP; + +typedef struct { + Eterm* ptr; + int i; +} ErtsMakeHash2Context_MAP_SUBTAG; + +typedef struct { + Uint num_free; + Eterm* bptr; +} ErtsMakeHash2Context_FUN_SUBTAG; + +typedef struct { + Eterm* ptr; + Uint i; + Uint n; + Uint32 con; +} ErtsMakeHash2Context_NEG_BIG_SUBTAG; + +typedef struct { + byte* bptr; + Uint sz; + Uint bitsize; + Uint bitoffs; + Uint no_bytes_processed; + ErtsBlockHashHelperCtx block_hash_ctx; + /* The following fields are only used when bitoffs != 0 */ + byte* buf; + int done; + +} ErtsMakeHash2Context_SUB_BINARY_SUBTAG; + +typedef struct { + int dummy__; /* Empty structs are not supported on all platforms */ +} ErtsMakeHash2Context_EMPTY; + +typedef struct { + ErtsMakeHash2TrapLocation trap_location; + /* specific to the trap location: */ + union { + ErtsMakeHash2Context_TAG_PRIMARY_LIST tag_primary_list; + ErtsMakeHash2Context_ARITYVAL_SUBTAG arityval_subtag; + ErtsMakeHash2Context_HAMT_SUBTAG_HEAD_FLATMAP hamt_subtag_head_flatmap; + ErtsMakeHash2Context_MAP_SUBTAG map_subtag; + ErtsMakeHash2Context_FUN_SUBTAG fun_subtag; + ErtsMakeHash2Context_NEG_BIG_SUBTAG neg_big_subtag; + ErtsMakeHash2Context_SUB_BINARY_SUBTAG sub_binary_subtag_1; + ErtsMakeHash2Context_SUB_BINARY_SUBTAG sub_binary_subtag_2; + ErtsMakeHash2Context_EMPTY hash2_common_1; + ErtsMakeHash2Context_EMPTY hash2_common_2; + ErtsMakeHash2Context_EMPTY hash2_common_3; + } trap_location_state; + /* same for all trap locations: */ + Eterm term; Uint32 hash; Uint32 hash_xor_pairs; - DeclareTmpHeapNoproc(tmp_big,2); + ErtsEStack stack; +} ErtsMakeHash2Context; + +static int make_hash2_ctx_bin_dtor(Binary *context_bin) { + ErtsMakeHash2Context* context = ERTS_MAGIC_BIN_DATA(context_bin); + DESTROY_SAVED_ESTACK(&context->stack); + if (context->trap_location == sub_binary_subtag_2 && + context->trap_location_state.sub_binary_subtag_2.buf != NULL) { + erts_free(ERTS_ALC_T_PHASH2_TRAP, context->trap_location_state.sub_binary_subtag_2.buf); + } + return 1; +} +/* hash2_save_trap_state is called seldom so we want to avoid inlining */ +static ERTS_NOINLINE +Eterm hash2_save_trap_state(Eterm state_mref, + Uint32 hash_xor_pairs, + Uint32 hash, + Process* p, + Eterm term, + Eterm* ESTK_DEF_STACK(s), + ErtsEStack s, + ErtsMakeHash2TrapLocation trap_location, + void* trap_location_state_ptr, + size_t trap_location_state_size) { + Binary* state_bin; + ErtsMakeHash2Context* context; + if (state_mref == THE_NON_VALUE) { + Eterm* hp; + state_bin = erts_create_magic_binary(sizeof(ErtsMakeHash2Context), + make_hash2_ctx_bin_dtor); + hp = HAlloc(p, ERTS_MAGIC_REF_THING_SIZE); + state_mref = erts_mk_magic_ref(&hp, &MSO(p), state_bin); + } else { + state_bin = erts_magic_ref2bin(state_mref); + } + context = ERTS_MAGIC_BIN_DATA(state_bin); + context->term = term; + context->hash = hash; + context->hash_xor_pairs = hash_xor_pairs; + ESTACK_SAVE(s, &context->stack); + context->trap_location = trap_location; + sys_memcpy(&context->trap_location_state, + trap_location_state_ptr, + trap_location_state_size); + erts_set_gc_state(p, 0); + BUMP_ALL_REDS(p); + return state_mref; +} +#undef NOINLINE_HASH2_SAVE_TRAP_STATE + +/* Writes back a magic reference to *state_mref_write_back when the + function traps */ +static ERTS_INLINE Uint32 +make_hash2_helper(Eterm term_param, const int can_trap, Eterm* state_mref_write_back, Process* p) +{ + static const Uint ITERATIONS_PER_RED = 64; + Uint32 hash; + Uint32 hash_xor_pairs; + Eterm term = term_param; ERTS_UNDEF(hash_xor_pairs, 0); /* (HCONST * {2, ..., 22}) mod 2^32 */ @@ -1168,12 +1351,63 @@ make_hash2(Eterm term) #define IS_SSMALL28(x) (((Uint) (((x) >> (28-1)) + 1)) < 2) +#define NOT_SSMALL28_HASH(SMALL) \ + do { \ + Uint64 t; \ + Uint32 x, y; \ + Uint32 con; \ + if (SMALL < 0) { \ + con = HCONST_10; \ + t = (Uint64)(SMALL * (-1)); \ + } else { \ + con = HCONST_11; \ + t = SMALL; \ + } \ + x = t & 0xffffffff; \ + y = t >> 32; \ + UINT32_HASH_2(x, y, con); \ + } while(0) + #ifdef ARCH_64 # define POINTER_HASH(Ptr, AConst) UINT32_HASH_2((Uint32)(UWord)(Ptr), (((UWord)(Ptr)) >> 32), AConst) #else # define POINTER_HASH(Ptr, AConst) UINT32_HASH(Ptr, AConst) #endif +#define TRAP_LOCATION_NO_RED(location_name) \ + do { \ + if(can_trap && iterations_until_trap <= 0) { \ + *state_mref_write_back = \ + hash2_save_trap_state(state_mref, \ + hash_xor_pairs, \ + hash, \ + p, \ + term, \ + ESTK_DEF_STACK(s), \ + s, \ + location_name, \ + &ctx, \ + sizeof(ctx)); \ + return 0; \ + L_##location_name: \ + ctx = context->trap_location_state. location_name; \ + } \ + } while(0) + +#define TRAP_LOCATION(location_name) \ + do { \ + if (can_trap) { \ + iterations_until_trap--; \ + TRAP_LOCATION_NO_RED(location_name); \ + } \ + } while(0) + +#define TRAP_LOCATION_NO_CTX(location_name) \ + do { \ + ErtsMakeHash2Context_EMPTY ctx; \ + TRAP_LOCATION(location_name); \ + } while(0) + /* Optimization. Simple cases before declaration of estack. */ if (primary_tag(term) == TAG_PRIMARY_IMMED1) { switch (term & _TAG_IMMED1_MASK) { @@ -1186,51 +1420,94 @@ make_hash2(Eterm term) break; case _TAG_IMMED1_SMALL: { - Sint x = signed_val(term); - - if (SMALL_BITS > 28 && !IS_SSMALL28(x)) { - term = small_to_big(x, tmp_big); - break; + Sint small = signed_val(term); + if (SMALL_BITS > 28 && !IS_SSMALL28(small)) { + hash = 0; + NOT_SSMALL28_HASH(small); + return hash; } hash = 0; - SINT32_HASH(x, HCONST); + SINT32_HASH(small, HCONST); return hash; } } }; { Eterm tmp; + long max_iterations = 0; + long iterations_until_trap = 0; + Eterm state_mref = THE_NON_VALUE; + ErtsMakeHash2Context* context = NULL; DECLARE_ESTACK(s); - - UseTmpHeapNoproc(2); + ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); + if(can_trap){ +#ifdef DEBUG + (void)ITERATIONS_PER_RED; + iterations_until_trap = max_iterations = + (1103515245 * (ERTS_BIF_REDS_LEFT(p)) + 12345) % 227; +#else + iterations_until_trap = max_iterations = + ITERATIONS_PER_RED * ERTS_BIF_REDS_LEFT(p); +#endif + } + if (can_trap && is_internal_magic_ref(term)) { + Binary* state_bin; + state_mref = term; + state_bin = erts_magic_ref2bin(state_mref); + if (ERTS_MAGIC_BIN_DESTRUCTOR(state_bin) == make_hash2_ctx_bin_dtor) { + /* Restore state after a trap */ + context = ERTS_MAGIC_BIN_DATA(state_bin); + term = context->term; + hash = context->hash; + hash_xor_pairs = context->hash_xor_pairs; + ESTACK_RESTORE(s, &context->stack); + ASSERT(p->flags & F_DISABLE_GC); + erts_set_gc_state(p, 1); + switch (context->trap_location) { + case hash2_common_3: goto L_hash2_common_3; + case tag_primary_list: goto L_tag_primary_list; + case arityval_subtag: goto L_arityval_subtag; + case hamt_subtag_head_flatmap: goto L_hamt_subtag_head_flatmap; + case map_subtag: goto L_map_subtag; + case fun_subtag: goto L_fun_subtag; + case neg_big_subtag: goto L_neg_big_subtag; + case sub_binary_subtag_1: goto L_sub_binary_subtag_1; + case sub_binary_subtag_2: goto L_sub_binary_subtag_2; + case hash2_common_1: goto L_hash2_common_1; + case hash2_common_2: goto L_hash2_common_2; + } + } + } hash = 0; for (;;) { switch (primary_tag(term)) { case TAG_PRIMARY_LIST: { - int c = 0; - Uint32 sh = 0; - Eterm* ptr = list_val(term); - while (is_byte(*ptr)) { + ErtsMakeHash2Context_TAG_PRIMARY_LIST ctx = { + .c = 0, + .sh = 0, + .ptr = list_val(term)}; + while (is_byte(*ctx.ptr)) { /* Optimization for strings. */ - sh = (sh << 8) + unsigned_val(*ptr); - if (c == 3) { - UINT32_HASH(sh, HCONST_4); - c = sh = 0; + ctx.sh = (ctx.sh << 8) + unsigned_val(*ctx.ptr); + if (ctx.c == 3) { + UINT32_HASH(ctx.sh, HCONST_4); + ctx.c = ctx.sh = 0; } else { - c++; + ctx.c++; } - term = CDR(ptr); + term = CDR(ctx.ptr); if (is_not_list(term)) break; - ptr = list_val(term); + ctx.ptr = list_val(term); + TRAP_LOCATION(tag_primary_list); } - if (c > 0) - UINT32_HASH(sh, HCONST_4); + if (ctx.c > 0) + UINT32_HASH(ctx.sh, HCONST_4); if (is_list(term)) { - tmp = CDR(ptr); + tmp = CDR(ctx.ptr); ESTACK_PUSH(s, tmp); - term = CAR(ptr); + term = CAR(ctx.ptr); } } break; @@ -1241,34 +1518,39 @@ make_hash2(Eterm term) switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: { - int i; - int arity = header_arity(hdr); - Eterm* elem = tuple_val(term); - UINT32_HASH(arity, HCONST_9); - if (arity == 0) /* Empty tuple */ + ErtsMakeHash2Context_ARITYVAL_SUBTAG ctx = { + .i = 0, + .arity = header_arity(hdr), + .elem = tuple_val(term)}; + UINT32_HASH(ctx.arity, HCONST_9); + if (ctx.arity == 0) /* Empty tuple */ goto hash2_common; - for (i = arity; ; i--) { - term = elem[i]; - if (i == 1) + for (ctx.i = ctx.arity; ; ctx.i--) { + term = ctx.elem[ctx.i]; + if (ctx.i == 1) break; ESTACK_PUSH(s, term); + TRAP_LOCATION(arityval_subtag); } } break; case MAP_SUBTAG: { - Eterm* ptr = boxed_val(term) + 1; Uint size; - int i; + ErtsMakeHash2Context_MAP_SUBTAG ctx = { + .ptr = boxed_val(term) + 1, + .i = 0}; switch (hdr & _HEADER_MAP_SUBTAG_MASK) { case HAMT_SUBTAG_HEAD_FLATMAP: { flatmap_t *mp = (flatmap_t *)flatmap_val(term); - Eterm *ks = flatmap_get_keys(mp); - Eterm *vs = flatmap_get_values(mp); - size = flatmap_get_size(mp); - UINT32_HASH(size, HCONST_16); - if (size == 0) + ErtsMakeHash2Context_HAMT_SUBTAG_HEAD_FLATMAP ctx = { + .ks = flatmap_get_keys(mp), + .vs = flatmap_get_values(mp), + .i = 0, + .size = flatmap_get_size(mp)}; + UINT32_HASH(ctx.size, HCONST_16); + if (ctx.size == 0) goto hash2_common; /* We want a portable hash function that is *independent* of @@ -1281,17 +1563,18 @@ make_hash2(Eterm term) ESTACK_PUSH(s, HASH_MAP_TAIL); hash = 0; hash_xor_pairs = 0; - for (i = size - 1; i >= 0; i--) { + for (ctx.i = ctx.size - 1; ctx.i >= 0; ctx.i--) { ESTACK_PUSH(s, HASH_MAP_PAIR); - ESTACK_PUSH(s, vs[i]); - ESTACK_PUSH(s, ks[i]); + ESTACK_PUSH(s, ctx.vs[ctx.i]); + ESTACK_PUSH(s, ctx.ks[ctx.i]); + TRAP_LOCATION(hamt_subtag_head_flatmap); } goto hash2_common; } case HAMT_SUBTAG_HEAD_ARRAY: case HAMT_SUBTAG_HEAD_BITMAP: - size = *ptr++; + size = *ctx.ptr++; UINT32_HASH(size, HCONST_16); if (size == 0) goto hash2_common; @@ -1303,27 +1586,28 @@ make_hash2(Eterm term) } switch (hdr & _HEADER_MAP_SUBTAG_MASK) { case HAMT_SUBTAG_HEAD_ARRAY: - i = 16; + ctx.i = 16; break; case HAMT_SUBTAG_HEAD_BITMAP: case HAMT_SUBTAG_NODE_BITMAP: - i = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + ctx.i = hashmap_bitcount(MAP_HEADER_VAL(hdr)); break; default: erts_exit(ERTS_ERROR_EXIT, "bad header"); } - while (i) { - if (is_list(*ptr)) { - Eterm* cons = list_val(*ptr); + while (ctx.i) { + if (is_list(*ctx.ptr)) { + Eterm* cons = list_val(*ctx.ptr); ESTACK_PUSH(s, HASH_MAP_PAIR); ESTACK_PUSH(s, CDR(cons)); ESTACK_PUSH(s, CAR(cons)); } else { - ASSERT(is_boxed(*ptr)); - ESTACK_PUSH(s, *ptr); + ASSERT(is_boxed(*ctx.ptr)); + ESTACK_PUSH(s, *ctx.ptr); } - i--; ptr++; + ctx.i--; ctx.ptr++; + TRAP_LOCATION(map_subtag); } goto hash2_common; } @@ -1344,22 +1628,25 @@ make_hash2(Eterm term) case FUN_SUBTAG: { ErlFunThing* funp = (ErlFunThing *) fun_val(term); - Uint num_free = funp->num_free; + ErtsMakeHash2Context_FUN_SUBTAG ctx = { + .num_free = funp->num_free, + .bptr = NULL}; UINT32_HASH_2 - (num_free, + (ctx.num_free, atom_tab(atom_val(funp->fe->module))->slot.bucket.hvalue, HCONST); UINT32_HASH_2 - (funp->fe->old_index, funp->fe->old_uniq, HCONST); - if (num_free == 0) { + (funp->fe->index, funp->fe->old_uniq, HCONST); + if (ctx.num_free == 0) { goto hash2_common; } else { - Eterm* bptr = funp->env + num_free - 1; - while (num_free-- > 1) { - term = *bptr--; + ctx.bptr = funp->env + ctx.num_free - 1; + while (ctx.num_free-- > 1) { + term = *ctx.bptr--; ESTACK_PUSH(s, term); + TRAP_LOCATION(fun_subtag); } - term = *bptr; + term = *ctx.bptr; } } break; @@ -1367,70 +1654,190 @@ make_hash2(Eterm term) case HEAP_BINARY_SUBTAG: case SUB_BINARY_SUBTAG: { - byte* bptr; - unsigned sz = binary_size(term); +#define BYTE_BITS 8 + ErtsMakeHash2Context_SUB_BINARY_SUBTAG ctx = { + .bptr = 0, + /* !!!!!!!!!!!!!!!!!!!! OBS !!!!!!!!!!!!!!!!!!!! + * + * The size is truncated to 32 bits on the line + * below so that the code is compatible with old + * versions of the code. This means that hash + * values for binaries with a size greater than + * 4GB do not take all bytes in consideration. + * + * !!!!!!!!!!!!!!!!!!!! OBS !!!!!!!!!!!!!!!!!!!! + */ + .sz = (0xFFFFFFFF & binary_size(term)), + .bitsize = 0, + .bitoffs = 0, + .no_bytes_processed = 0 + }; Uint32 con = HCONST_13 + hash; - Uint bitoffs; - Uint bitsize; - - ERTS_GET_BINARY_BYTES(term, bptr, bitoffs, bitsize); - if (sz == 0 && bitsize == 0) { + Uint iters_for_bin = MAX(1, ctx.sz / BLOCK_HASH_BYTES_PER_ITER); + ERTS_GET_BINARY_BYTES(term, ctx.bptr, ctx.bitoffs, ctx.bitsize); + if (ctx.sz == 0 && ctx.bitsize == 0) { hash = con; - } else { - if (bitoffs == 0) { - hash = block_hash(bptr, sz, con); - if (bitsize > 0) { - UINT32_HASH_2(bitsize, (bptr[sz] >> (8 - bitsize)), - HCONST_15); - } - } else { - byte* buf = (byte *) erts_alloc(ERTS_ALC_T_TMP, - sz + (bitsize != 0)); - erts_copy_bits(bptr, bitoffs, 1, buf, 0, 1, sz*8+bitsize); - hash = block_hash(buf, sz, con); - if (bitsize > 0) { - UINT32_HASH_2(bitsize, (buf[sz] >> (8 - bitsize)), - HCONST_15); - } - erts_free(ERTS_ALC_T_TMP, (void *) buf); - } + } else if (ctx.bitoffs == 0 && + (!can_trap || + (iterations_until_trap - iters_for_bin) > 0)) { + /* No need to trap while hashing binary */ + if (can_trap) iterations_until_trap -= iters_for_bin; + hash = block_hash(ctx.bptr, ctx.sz, con); + if (ctx.bitsize > 0) { + UINT32_HASH_2(ctx.bitsize, + (ctx.bptr[ctx.sz] >> (BYTE_BITS - ctx.bitsize)), + HCONST_15); + } + } else if (ctx.bitoffs == 0) { + /* Need to trap while hashing binary */ + ErtsBlockHashHelperCtx* block_hash_ctx = &ctx.block_hash_ctx; + block_hash_setup(con, block_hash_ctx); + do { + Uint max_bytes_to_process = + iterations_until_trap <= 0 ? BLOCK_HASH_BYTES_PER_ITER : + iterations_until_trap * BLOCK_HASH_BYTES_PER_ITER; + Uint bytes_left = ctx.sz - ctx.no_bytes_processed; + Uint even_bytes_left = + bytes_left - (bytes_left % BLOCK_HASH_BYTES_PER_ITER); + Uint bytes_to_process = + MIN(max_bytes_to_process, even_bytes_left); + block_hash_buffer(&ctx.bptr[ctx.no_bytes_processed], + bytes_to_process, + block_hash_ctx); + ctx.no_bytes_processed += bytes_to_process; + iterations_until_trap -= + MAX(1, bytes_to_process / BLOCK_HASH_BYTES_PER_ITER); + TRAP_LOCATION_NO_RED(sub_binary_subtag_1); + block_hash_ctx = &ctx.block_hash_ctx; /* Restore after trap */ + } while ((ctx.sz - ctx.no_bytes_processed) >= + BLOCK_HASH_BYTES_PER_ITER); + hash = block_hash_final_bytes(ctx.bptr + + ctx.no_bytes_processed, + ctx.sz - ctx.no_bytes_processed, + ctx.sz, + block_hash_ctx); + if (ctx.bitsize > 0) { + UINT32_HASH_2(ctx.bitsize, + (ctx.bptr[ctx.sz] >> (BYTE_BITS - ctx.bitsize)), + HCONST_15); + } + } else if (/* ctx.bitoffs != 0 && */ + (!can_trap || + (iterations_until_trap - iters_for_bin) > 0)) { + /* No need to trap while hashing binary */ + Uint nr_of_bytes = ctx.sz + (ctx.bitsize != 0); + byte *buf = erts_alloc(ERTS_ALC_T_TMP, nr_of_bytes); + Uint nr_of_bits_to_copy = ctx.sz*BYTE_BITS+ctx.bitsize; + if (can_trap) iterations_until_trap -= iters_for_bin; + erts_copy_bits(ctx.bptr, + ctx.bitoffs, 1, buf, 0, 1, nr_of_bits_to_copy); + hash = block_hash(buf, ctx.sz, con); + if (ctx.bitsize > 0) { + UINT32_HASH_2(ctx.bitsize, + (buf[ctx.sz] >> (BYTE_BITS - ctx.bitsize)), + HCONST_15); + } + erts_free(ERTS_ALC_T_TMP, buf); + } else /* ctx.bitoffs != 0 && */ { +#ifdef DEBUG +#define BINARY_BUF_SIZE (BLOCK_HASH_BYTES_PER_ITER * 3) +#else +#define BINARY_BUF_SIZE (BLOCK_HASH_BYTES_PER_ITER * 256) +#endif +#define BINARY_BUF_SIZE_BITS (BINARY_BUF_SIZE*BYTE_BITS) + /* Need to trap while hashing binary */ + ErtsBlockHashHelperCtx* block_hash_ctx = &ctx.block_hash_ctx; + Uint nr_of_bytes = ctx.sz + (ctx.bitsize != 0); + ERTS_CT_ASSERT(BINARY_BUF_SIZE % BLOCK_HASH_BYTES_PER_ITER == 0); + ctx.buf = erts_alloc(ERTS_ALC_T_PHASH2_TRAP, + MIN(nr_of_bytes, BINARY_BUF_SIZE)); + block_hash_setup(con, block_hash_ctx); + do { + Uint bytes_left = + ctx.sz - ctx.no_bytes_processed; + Uint even_bytes_left = + bytes_left - (bytes_left % BLOCK_HASH_BYTES_PER_ITER); + Uint bytes_to_process = + MIN(BINARY_BUF_SIZE, even_bytes_left); + Uint nr_of_bits_left = + (ctx.sz*BYTE_BITS+ctx.bitsize) - + ctx.no_bytes_processed*BYTE_BITS; + Uint nr_of_bits_to_copy = + MIN(nr_of_bits_left, BINARY_BUF_SIZE_BITS); + ctx.done = nr_of_bits_left == nr_of_bits_to_copy; + erts_copy_bits(ctx.bptr + ctx.no_bytes_processed, + ctx.bitoffs, 1, ctx.buf, 0, 1, + nr_of_bits_to_copy); + block_hash_buffer(ctx.buf, + bytes_to_process, + block_hash_ctx); + ctx.no_bytes_processed += bytes_to_process; + iterations_until_trap -= + MAX(1, bytes_to_process / BLOCK_HASH_BYTES_PER_ITER); + TRAP_LOCATION_NO_RED(sub_binary_subtag_2); + block_hash_ctx = &ctx.block_hash_ctx; /* Restore after trap */ + } while (!ctx.done); + nr_of_bytes = ctx.sz + (ctx.bitsize != 0); + hash = block_hash_final_bytes(ctx.buf + + (ctx.no_bytes_processed - + ((nr_of_bytes-1) / BINARY_BUF_SIZE) * BINARY_BUF_SIZE), + ctx.sz - ctx.no_bytes_processed, + ctx.sz, + block_hash_ctx); + if (ctx.bitsize > 0) { + Uint last_byte_index = + nr_of_bytes - (((nr_of_bytes-1) / BINARY_BUF_SIZE) * BINARY_BUF_SIZE) -1; + UINT32_HASH_2(ctx.bitsize, + (ctx.buf[last_byte_index] >> (BYTE_BITS - ctx.bitsize)), + HCONST_15); + } + erts_free(ERTS_ALC_T_PHASH2_TRAP, ctx.buf); + context->trap_location_state.sub_binary_subtag_2.buf = NULL; } goto hash2_common; +#undef BYTE_BITS +#undef BINARY_BUF_SIZE +#undef BINARY_BUF_SIZE_BITS } break; case POS_BIG_SUBTAG: case NEG_BIG_SUBTAG: { - Eterm* ptr = big_val(term); - Uint i = 0; - Uint n = BIG_SIZE(ptr); - Uint32 con = BIG_SIGN(ptr) ? HCONST_10 : HCONST_11; + Eterm* big_val_ptr = big_val(term); + ErtsMakeHash2Context_NEG_BIG_SUBTAG ctx = { + .ptr = big_val_ptr, + .i = 0, + .n = BIG_SIZE(big_val_ptr), + .con = BIG_SIGN(big_val_ptr) ? HCONST_10 : HCONST_11}; #if D_EXP == 16 do { Uint32 x, y; - x = i < n ? BIG_DIGIT(ptr, i++) : 0; - x += (Uint32)(i < n ? BIG_DIGIT(ptr, i++) : 0) << 16; - y = i < n ? BIG_DIGIT(ptr, i++) : 0; - y += (Uint32)(i < n ? BIG_DIGIT(ptr, i++) : 0) << 16; - UINT32_HASH_2(x, y, con); - } while (i < n); + x = ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0; + x += (Uint32)(ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0) << 16; + y = ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0; + y += (Uint32)(ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0) << 16; + UINT32_HASH_2(x, y, ctx.con); + TRAP_LOCATION(neg_big_subtag); + } while (ctx.i < ctx.n); #elif D_EXP == 32 do { Uint32 x, y; - x = i < n ? BIG_DIGIT(ptr, i++) : 0; - y = i < n ? BIG_DIGIT(ptr, i++) : 0; - UINT32_HASH_2(x, y, con); - } while (i < n); + x = ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0; + y = ctx.i < ctx.n ? BIG_DIGIT(ctx.ptr, ctx.i++) : 0; + UINT32_HASH_2(x, y, ctx.con); + TRAP_LOCATION(neg_big_subtag); + } while (ctx.i < ctx.n); #elif D_EXP == 64 do { Uint t; Uint32 x, y; - ASSERT(i < n); - t = BIG_DIGIT(ptr, i++); + ASSERT(ctx.i < ctx.n); + t = BIG_DIGIT(ctx.ptr, ctx.i++); x = t & 0xffffffff; y = t >> 32; - UINT32_HASH_2(x, y, con); - } while (i < n); + UINT32_HASH_2(x, y, ctx.con); + TRAP_LOCATION(neg_big_subtag); + } while (ctx.i < ctx.n); #else #error "unsupported D_EXP size" #endif @@ -1508,13 +1915,13 @@ make_hash2(Eterm term) } case _TAG_IMMED1_SMALL: { - Sint x = signed_val(term); + Sint small = signed_val(term); + if (SMALL_BITS > 28 && !IS_SSMALL28(small)) { + NOT_SSMALL28_HASH(small); + } else { + SINT32_HASH(small, HCONST); + } - if (SMALL_BITS > 28 && !IS_SSMALL28(x)) { - term = small_to_big(x, tmp_big); - break; - } - SINT32_HASH(x, HCONST); goto hash2_common; } } @@ -1529,7 +1936,10 @@ make_hash2(Eterm term) if (ESTACK_ISEMPTY(s)) { DESTROY_ESTACK(s); - UnUseTmpHeapNoproc(2); + if (can_trap) { + BUMP_REDS(p, (max_iterations - iterations_until_trap) / ITERATIONS_PER_RED); + ASSERT(!(p->flags & F_DISABLE_GC)); + } return hash; } @@ -1540,18 +1950,37 @@ make_hash2(Eterm term) hash = (Uint32) ESTACK_POP(s); UINT32_HASH(hash_xor_pairs, HCONST_19); hash_xor_pairs = (Uint32) ESTACK_POP(s); + TRAP_LOCATION_NO_CTX(hash2_common_1); goto hash2_common; } case HASH_MAP_PAIR: hash_xor_pairs ^= hash; hash = 0; + TRAP_LOCATION_NO_CTX(hash2_common_2); goto hash2_common; default: break; } + } + TRAP_LOCATION_NO_CTX(hash2_common_3); } } +#undef TRAP_LOCATION_NO_RED +#undef TRAP_LOCATION +#undef TRAP_LOCATION_NO_CTX +} + +Uint32 +make_hash2(Eterm term) +{ + return make_hash2_helper(term, 0, NULL, NULL); +} + +Uint32 +trapping_make_hash2(Eterm term, Eterm* state_mref_write_back, Process* p) +{ + return make_hash2_helper(term, 1, state_mref_write_back, p); } /* Term hash function for internal use. @@ -1731,7 +2160,7 @@ make_internal_hash(Eterm term, Uint32 salt) ErlFunThing* funp = (ErlFunThing *) fun_val(term); Uint num_free = funp->num_free; UINT32_HASH_2(num_free, funp->fe->module, HCONST_20); - UINT32_HASH_2(funp->fe->old_index, funp->fe->old_uniq, HCONST_21); + UINT32_HASH_2(funp->fe->index, funp->fe->old_uniq, HCONST_21); if (num_free == 0) { goto pop_next; } else { @@ -2381,7 +2810,7 @@ tailrecur_ne: f1 = (ErlFunThing *) fun_val(a); f2 = (ErlFunThing *) fun_val(b); if (f1->fe->module != f2->fe->module || - f1->fe->old_index != f2->fe->old_index || + f1->fe->index != f2->fe->index || f1->fe->old_uniq != f2->fe->old_uniq || f1->num_free != f2->num_free) { goto not_equal; @@ -2976,7 +3405,7 @@ tailrecur_ne: if (diff != 0) { RETURN_NEQ(diff); } - diff = f1->fe->old_index - f2->fe->old_index; + diff = f1->fe->index - f2->fe->index; if (diff != 0) { RETURN_NEQ(diff); } diff --git a/erts/emulator/hipe/hipe_debug.c b/erts/emulator/hipe/hipe_debug.c index 138e4f7da3..2e34cfac59 100644 --- a/erts/emulator/hipe/hipe_debug.c +++ b/erts/emulator/hipe/hipe_debug.c @@ -232,7 +232,6 @@ void hipe_print_pcb(Process *p) U("intial.fun ", u.initial.function); U("intial.ari ", u.initial.arity); U("current ", current); - P("cp ", cp); P("i ", i); U("catches ", catches); U("arity ", arity); diff --git a/erts/emulator/hipe/hipe_instrs.tab b/erts/emulator/hipe/hipe_instrs.tab index a01baebddf..8aa8544b2a 100644 --- a/erts/emulator/hipe/hipe_instrs.tab +++ b/erts/emulator/hipe/hipe_instrs.tab @@ -86,15 +86,14 @@ hipe_trap.post() { switch( c_p->def_arg_reg[3] ) { case HIPE_MODE_SWITCH_RES_RETURN: ASSERT(is_value(reg[0])); - SET_I(c_p->cp); - c_p->cp = 0; + $RETURN(); Goto(*I); case HIPE_MODE_SWITCH_RES_CALL_EXPORTED: c_p->i = c_p->hipe.u.callee_exp->addressv[erts_active_code_ix()]; /*fall through*/ case HIPE_MODE_SWITCH_RES_CALL_BEAM: SET_I(c_p->i); - Dispatch(); + $DISPATCH(); case HIPE_MODE_SWITCH_RES_CALL_CLOSURE: /* This can be used to call any function value, but currently it's only used to call closures referring to unloaded @@ -105,13 +104,11 @@ hipe_trap.post() { next = call_fun(c_p, c_p->arity - 1, reg, THE_NON_VALUE); HEAVY_SWAPIN; if (next != NULL) { - SET_I(next); - Dispatchfun(); + $DISPATCH_FUN(next); } goto find_func_info; } case HIPE_MODE_SWITCH_RES_THROW: - c_p->cp = NULL; I = handle_error(c_p, I, reg, NULL); goto post_error_handling; default: diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c index 052cf9c263..863c5e6d44 100644 --- a/erts/emulator/hipe/hipe_mode_switch.c +++ b/erts/emulator/hipe/hipe_mode_switch.c @@ -202,15 +202,13 @@ hipe_push_beam_trap_frame(Process *p, Eterm reg[], unsigned arity) p->stop -= 2; p->stop[1] = hipe_beam_catch_throw; } - p->stop[0] = make_cp(p->cp); + p->stop[0] = (BeamInstr) hipe_beam_pc_return; ++p->catches; - p->cp = hipe_beam_pc_return; } static __inline__ void hipe_pop_beam_trap_frame(Process *p) { ASSERT(p->stop[1] == hipe_beam_catch_throw); - p->cp = cp_val(p->stop[0]); --p->catches; p->stop += 2; } @@ -263,7 +261,7 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[]) unsigned arity = cmd >> 8; /* p->hipe.u.ncallee set in beam_emu */ - if (p->cp == hipe_beam_pc_return) { + if (cp_val(p->stop[0]) == hipe_beam_pc_return) { /* Native called BEAM, which now tailcalls native. */ hipe_pop_beam_trap_frame(p); result = hipe_tailcall_to_native(p, arity, reg); @@ -292,7 +290,7 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[]) /* just like a normal call from now on */ /* p->hipe.u.ncallee set in beam_emu */ - if (p->cp == hipe_beam_pc_return) { + if (cp_val(p->stop[0]) == hipe_beam_pc_return) { /* Native called BEAM, which now tailcalls native. */ hipe_pop_beam_trap_frame(p); result = hipe_tailcall_to_native(p, arity, reg); diff --git a/erts/emulator/internal_doc/beam_makeops.md b/erts/emulator/internal_doc/beam_makeops.md index 2880099b70..267af78412 100644 --- a/erts/emulator/internal_doc/beam_makeops.md +++ b/erts/emulator/internal_doc/beam_makeops.md @@ -1457,26 +1457,26 @@ all instructions. It expands to the address of the next instruction. Here is an example: i_call(CallDest) { - SET_CP(c_p, $NEXT_INSTRUCTION); + //| -no_next + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); $DISPATCH_REL($CallDest); } -When calling a function, the return address is first stored in `c_p->cp` -(using the `SET_CP()` macro defined in `beam_emu.c`), and then control is +When calling a function, the return address is first stored in `E[0]` +(using the `$SAVE_CONTINUATION_POINTER()` macro), and then control is transferred to the callee. Here is the generated code: OpCase(i_call_f): { - SET_CP(c_p, I+1); - ASSERT(VALID_INSTR(*(I + (fb(BeamExtraData(I[0]))) + 0))); - I += fb(BeamExtraData(I[0])) + 0;; - DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); - Dispatch();; + ASSERT(VALID_INSTR(*(I+2))); + *E = (BeamInstr) (I+2);; + + /* ... dispatch code intentionally left out ... */ } -We can see that that `$NEXT_INSTRUCTION` has been expanded to `I+1`. +We can see that that `$NEXT_INSTRUCTION` has been expanded to `I+2`. That makes sense since the size of the `i_call_f/1` instruction is -one word. +two words. ##### The IP_ADJUSTMENT pre-bound variable ##### diff --git a/erts/emulator/nifs/common/prim_file_nif.c b/erts/emulator/nifs/common/prim_file_nif.c index 3df04e42e2..5c5e9a2d30 100644 --- a/erts/emulator/nifs/common/prim_file_nif.c +++ b/erts/emulator/nifs/common/prim_file_nif.c @@ -162,6 +162,7 @@ WRAP_FILE_HANDLE_EXPORT(allocate_nif) WRAP_FILE_HANDLE_EXPORT(advise_nif) WRAP_FILE_HANDLE_EXPORT(get_handle_nif) WRAP_FILE_HANDLE_EXPORT(ipread_s32bu_p32bu_nif) +WRAP_FILE_HANDLE_EXPORT(read_handle_info_nif) static ErlNifFunc nif_funcs[] = { /* File handle ops */ @@ -176,6 +177,7 @@ static ErlNifFunc nif_funcs[] = { {"truncate_nif", 1, truncate_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, {"allocate_nif", 3, allocate_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, {"advise_nif", 4, advise_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"read_handle_info_nif", 1, read_handle_info_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, /* Filesystem ops */ {"make_hard_link_nif", 2, make_hard_link_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, @@ -231,6 +233,7 @@ static int load(ErlNifEnv *env, void** priv_data, ERL_NIF_TERM prim_file_pid) am_append = enif_make_atom(env, "append"); am_sync = enif_make_atom(env, "sync"); am_skip_type_check = enif_make_atom(env, "skip_type_check"); + am_directory = enif_make_atom(env, "directory"); am_read_write = enif_make_atom(env, "read_write"); am_none = enif_make_atom(env, "none"); @@ -447,6 +450,8 @@ static enum efile_modes_t efile_translate_modelist(ErlNifEnv *env, ERL_NIF_TERM modes |= EFILE_MODE_SYNC; } else if(enif_is_identical(head, am_skip_type_check)) { modes |= EFILE_MODE_SKIP_TYPE_CHECK; + } else if (enif_is_identical(head, am_directory)) { + modes |= EFILE_MODE_DIRECTORY; } else { /* Modes like 'raw', 'ram', 'delayed_writes' etc are handled * further up the chain. */ @@ -893,6 +898,26 @@ static ERL_NIF_TERM get_handle_nif_impl(efile_data_t *d, ErlNifEnv *env, int arg return efile_get_handle(env, d); } +static ERL_NIF_TERM build_file_info(ErlNifEnv *env, efile_fileinfo_t *info) { + /* #file_info as declared in file.hrl */ + return enif_make_tuple(env, 14, + am_file_info, + enif_make_uint64(env, info->size), + efile_filetype_to_atom(info->type), + efile_access_to_atom(info->access), + enif_make_int64(env, MAX(EFILE_MIN_FILETIME, info->a_time)), + enif_make_int64(env, MAX(EFILE_MIN_FILETIME, info->m_time)), + enif_make_int64(env, MAX(EFILE_MIN_FILETIME, info->c_time)), + enif_make_uint(env, info->mode), + enif_make_uint(env, info->links), + enif_make_uint(env, info->major_device), + enif_make_uint(env, info->minor_device), + enif_make_uint(env, info->inode), + enif_make_uint(env, info->uid), + enif_make_uint(env, info->gid) + ); +} + static ERL_NIF_TERM read_info_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { posix_errno_t posix_errno; @@ -911,23 +936,20 @@ static ERL_NIF_TERM read_info_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM a return posix_error_to_tuple(env, posix_errno); } - /* #file_info as declared in file.hrl */ - return enif_make_tuple(env, 14, - am_file_info, - enif_make_uint64(env, info.size), - efile_filetype_to_atom(info.type), - efile_access_to_atom(info.access), - enif_make_int64(env, MAX(EFILE_MIN_FILETIME, info.a_time)), - enif_make_int64(env, MAX(EFILE_MIN_FILETIME, info.m_time)), - enif_make_int64(env, MAX(EFILE_MIN_FILETIME, info.c_time)), - enif_make_uint(env, info.mode), - enif_make_uint(env, info.links), - enif_make_uint(env, info.major_device), - enif_make_uint(env, info.minor_device), - enif_make_uint(env, info.inode), - enif_make_uint(env, info.uid), - enif_make_uint(env, info.gid) - ); + return build_file_info(env, &info); +} + +static ERL_NIF_TERM read_handle_info_nif_impl(efile_data_t *d, ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { + posix_errno_t posix_errno; + efile_fileinfo_t info = {0}; + + ASSERT(argc == 0); + + if((posix_errno = efile_read_handle_info(d, &info))) { + return posix_error_to_tuple(env, posix_errno); + } + + return build_file_info(env, &info); } static ERL_NIF_TERM set_permissions_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { diff --git a/erts/emulator/nifs/common/prim_file_nif.h b/erts/emulator/nifs/common/prim_file_nif.h index b2e30c59dd..28c1ea9d00 100644 --- a/erts/emulator/nifs/common/prim_file_nif.h +++ b/erts/emulator/nifs/common/prim_file_nif.h @@ -30,6 +30,8 @@ enum efile_modes_t { EFILE_MODE_SKIP_TYPE_CHECK = (1 << 5), /* Special for device files on Unix. */ EFILE_MODE_NO_TRUNCATE = (1 << 6), /* Special for reopening on VxWorks. */ + EFILE_MODE_DIRECTORY = (1 << 7), + EFILE_MODE_READ_WRITE = EFILE_MODE_READ | EFILE_MODE_WRITE }; @@ -168,6 +170,7 @@ int efile_close(efile_data_t *d, posix_errno_t *error); /* **** **** **** **** **** **** **** **** **** **** **** **** **** **** **** */ posix_errno_t efile_read_info(const efile_path_t *path, int follow_link, efile_fileinfo_t *result); +posix_errno_t efile_read_handle_info(efile_data_t *d, efile_fileinfo_t *result); /** @brief Sets the file times to the given values. Refer to efile_fileinfo_t * for a description of each. */ diff --git a/erts/emulator/nifs/common/socket_nif.c b/erts/emulator/nifs/common/socket_nif.c index bbeb8b6cdd..881a9c7ccd 100644 --- a/erts/emulator/nifs/common/socket_nif.c +++ b/erts/emulator/nifs/common/socket_nif.c @@ -9751,8 +9751,12 @@ ERL_NIF_TERM esock_setopt_lvl_ipv6_addrform(ErlNifEnv* env, domain) ); res = socket_setopt(descP->sock, - SOL_IPV6, IPV6_ADDRFORM, - &domain, sizeof(domain)); +#if defined(SOL_IPV6) + SOL_IPV6, +#else + IPPROTO_IPV6, +#endif + IPV6_ADDRFORM, &domain, sizeof(domain)); if (res != 0) result = esock_make_error_errno(env, sock_errno()); @@ -9782,7 +9786,14 @@ ERL_NIF_TERM esock_setopt_lvl_ipv6_authhdr(ErlNifEnv* env, ESockDescriptor* descP, ERL_NIF_TERM eVal) { - return esock_setopt_bool_opt(env, descP, SOL_IPV6, IPV6_AUTHHDR, eVal); +#if defined(SOL_IPV6) + int level = SOL_IPV6; +#else + int level = IPPROTO_IPV6; +#endif + + + return esock_setopt_bool_opt(env, descP, level, IPV6_AUTHHDR, eVal); } #endif diff --git a/erts/emulator/nifs/unix/unix_prim_file.c b/erts/emulator/nifs/unix/unix_prim_file.c index fbf312a3b8..e5099dd921 100644 --- a/erts/emulator/nifs/unix/unix_prim_file.c +++ b/erts/emulator/nifs/unix/unix_prim_file.c @@ -107,7 +107,7 @@ ERL_NIF_TERM efile_get_handle(ErlNifEnv *env, efile_data_t *d) { return result; } -static int open_file_type_check(const efile_path_t *path, int fd) { +static int open_file_is_dir(const efile_path_t *path, int fd) { struct stat file_info; int error; @@ -119,27 +119,14 @@ static int open_file_type_check(const efile_path_t *path, int fd) { (void)path; #endif - if(error < 0) { - /* If we failed to stat assume success and let the next call handle the - * error. The old driver checked whether the file was to be used - * immediately in a read within the call, but the new implementation - * never does that. */ - return 1; - } - - /* Allow everything that isn't a directory, and error out on the next call - * if it's unsupported. */ - if(S_ISDIR(file_info.st_mode)) { - return 0; - } - - return 1; + /* Assume not a directory on error. */ + return error == 0 && S_ISDIR(file_info.st_mode); } posix_errno_t efile_open(const efile_path_t *path, enum efile_modes_t modes, ErlNifResourceType *nif_type, efile_data_t **d) { - int flags, fd; + int mode, flags, fd; flags = 0; @@ -174,18 +161,38 @@ posix_errno_t efile_open(const efile_path_t *path, enum efile_modes_t modes, #endif } + if(modes & EFILE_MODE_DIRECTORY) { + mode = DIR_MODE; +#ifdef O_DIRECTORY + flags |= O_DIRECTORY; +#endif + } else { + mode = FILE_MODE; + } + do { - fd = open((const char*)path->data, flags, FILE_MODE); + fd = open((const char*)path->data, flags, mode); } while(fd == -1 && errno == EINTR); if(fd != -1) { efile_unix_t *u; - if(!(modes & EFILE_MODE_SKIP_TYPE_CHECK) && !open_file_type_check(path, fd)) { +#ifndef O_DIRECTORY + /* On platforms without O_DIRECTORY support, ensure that using the + * directory flag to open a file fails. */ + if(!(modes & EFILE_MODE_SKIP_TYPE_CHECK) && + (modes & EFILE_MODE_DIRECTORY) && !open_file_is_dir(path, fd)) { close(fd); + return ENOTDIR; + } +#endif - /* This is blatantly incorrect, but we're documented as returning - * this for everything that isn't a file. */ + /* open() works on directories without the O_DIRECTORY flag but for + * consistency across platforms we require that the user has requested + * directory mode. */ + if(!(modes & EFILE_MODE_SKIP_TYPE_CHECK) && + !(modes & EFILE_MODE_DIRECTORY) && open_file_is_dir(path, fd)) { + close(fd); return EISDIR; } @@ -656,6 +663,33 @@ int efile_truncate(efile_data_t *d) { return 1; } +static void build_file_info(struct stat *data, efile_fileinfo_t *result) { + if(S_ISCHR(data->st_mode) || S_ISBLK(data->st_mode)) { + result->type = EFILE_FILETYPE_DEVICE; + } else if(S_ISDIR(data->st_mode)) { + result->type = EFILE_FILETYPE_DIRECTORY; + } else if(S_ISREG(data->st_mode)) { + result->type = EFILE_FILETYPE_REGULAR; + } else if(S_ISLNK(data->st_mode)) { + result->type = EFILE_FILETYPE_SYMLINK; + } else { + result->type = EFILE_FILETYPE_OTHER; + } + + result->a_time = (Sint64)data->st_atime; + result->m_time = (Sint64)data->st_mtime; + result->c_time = (Sint64)data->st_ctime; + result->size = data->st_size; + + result->major_device = data->st_dev; + result->minor_device = data->st_rdev; + result->links = data->st_nlink; + result->inode = data->st_ino; + result->mode = data->st_mode; + result->uid = data->st_uid; + result->gid = data->st_gid; +} + posix_errno_t efile_read_info(const efile_path_t *path, int follow_links, efile_fileinfo_t *result) { struct stat data; @@ -669,30 +703,7 @@ posix_errno_t efile_read_info(const efile_path_t *path, int follow_links, efile_ } } - if(S_ISCHR(data.st_mode) || S_ISBLK(data.st_mode)) { - result->type = EFILE_FILETYPE_DEVICE; - } else if(S_ISDIR(data.st_mode)) { - result->type = EFILE_FILETYPE_DIRECTORY; - } else if(S_ISREG(data.st_mode)) { - result->type = EFILE_FILETYPE_REGULAR; - } else if(S_ISLNK(data.st_mode)) { - result->type = EFILE_FILETYPE_SYMLINK; - } else { - result->type = EFILE_FILETYPE_OTHER; - } - - result->a_time = (Sint64)data.st_atime; - result->m_time = (Sint64)data.st_mtime; - result->c_time = (Sint64)data.st_ctime; - result->size = data.st_size; - - result->major_device = data.st_dev; - result->minor_device = data.st_rdev; - result->links = data.st_nlink; - result->inode = data.st_ino; - result->mode = data.st_mode; - result->uid = data.st_uid; - result->gid = data.st_gid; + build_file_info(&data, result); #ifndef NO_ACCESS result->access = EFILE_ACCESS_NONE; @@ -711,6 +722,56 @@ posix_errno_t efile_read_info(const efile_path_t *path, int follow_links, efile_ return 0; } +static int check_access(struct stat *st) { + int ret = EFILE_ACCESS_NONE; + + if(st->st_uid == getuid()) { + if(st->st_mode & S_IRUSR) { + ret |= EFILE_ACCESS_READ; + } + if(st->st_mode & S_IWUSR) { + ret |= EFILE_ACCESS_WRITE; + } + return ret; + } + + if(st->st_gid == getgid()) { + if(st->st_mode & S_IRGRP) { + ret |= EFILE_ACCESS_READ; + } + if(st->st_mode & S_IWGRP) { + ret |= EFILE_ACCESS_WRITE; + } + return ret; + } + + if(st->st_mode & S_IROTH) { + ret |= EFILE_ACCESS_READ; + } + if(st->st_mode & S_IWOTH) { + ret |= EFILE_ACCESS_WRITE; + } + return ret; +} + +posix_errno_t efile_read_handle_info(efile_data_t *d, efile_fileinfo_t *result) { + struct stat data; + efile_unix_t *u = (efile_unix_t*)d; + +#ifdef HAVE_FSTAT + if(fstat(u->fd, &data) < 0) { + return errno; + } + + build_file_info(&data, result); + result->access = check_access(&data); + + return 0; +#else + return ENOTSUP; +#endif +} + posix_errno_t efile_set_permissions(const efile_path_t *path, Uint32 permissions) { const mode_t MUTABLE_MODES = (S_ISUID | S_ISGID | S_IRWXU | S_IRWXG | S_IRWXO); mode_t new_modes = permissions & MUTABLE_MODES; diff --git a/erts/emulator/nifs/win32/win_prim_file.c b/erts/emulator/nifs/win32/win_prim_file.c index e7d3924240..839ac3ea6e 100644 --- a/erts/emulator/nifs/win32/win_prim_file.c +++ b/erts/emulator/nifs/win32/win_prim_file.c @@ -270,6 +270,17 @@ static int normalize_path_result(ErlNifBinary *path) { } /* @brief Checks whether all the given attributes are set on the object at the + * given handle. Note that it assumes false on errors. */ +static int handle_has_file_attributes(HANDLE handle, DWORD mask) { + BY_HANDLE_FILE_INFORMATION native_file_info; + if(!GetFileInformationByHandle(handle, &native_file_info)) { + return 0; + } + + return !!((native_file_info.dwFileAttributes & mask) == mask); +} + +/* @brief Checks whether all the given attributes are set on the object at the * given path. Note that it assumes false on errors. */ static int has_file_attributes(const efile_path_t *path, DWORD mask) { DWORD attributes = GetFileAttributesW((WCHAR*)path->data); @@ -412,10 +423,15 @@ posix_errno_t efile_open(const efile_path_t *path, enum efile_modes_t modes, ASSERT_PATH_FORMAT(path); + attributes = 0; access_flags = 0; open_mode = 0; - if(modes & EFILE_MODE_READ && !(modes & EFILE_MODE_WRITE)) { + if(modes & EFILE_MODE_DIRECTORY) { + attributes = FILE_FLAG_BACKUP_SEMANTICS; + access_flags = GENERIC_READ; + open_mode = OPEN_EXISTING; + } else if(modes & EFILE_MODE_READ && !(modes & EFILE_MODE_WRITE)) { access_flags = GENERIC_READ; open_mode = OPEN_EXISTING; } else if(modes & EFILE_MODE_WRITE && !(modes & EFILE_MODE_READ)) { @@ -438,9 +454,9 @@ posix_errno_t efile_open(const efile_path_t *path, enum efile_modes_t modes, } if(modes & EFILE_MODE_SYNC) { - attributes = FILE_FLAG_WRITE_THROUGH; + attributes |= FILE_FLAG_WRITE_THROUGH; } else { - attributes = FILE_ATTRIBUTE_NORMAL; + attributes |= FILE_ATTRIBUTE_NORMAL; } handle = CreateFileW((WCHAR*)path->data, access_flags, @@ -449,6 +465,12 @@ posix_errno_t efile_open(const efile_path_t *path, enum efile_modes_t modes, if(handle != INVALID_HANDLE_VALUE) { efile_win_t *w; + /* Directory mode specified, but path is not a directory. */ + if((modes & EFILE_MODE_DIRECTORY) && !handle_has_file_attributes(handle, FILE_ATTRIBUTE_DIRECTORY)) { + CloseHandle(handle); + return ENOTDIR; + } + w = (efile_win_t*)enif_alloc_resource(nif_type, sizeof(efile_win_t)); w->handle = handle; @@ -461,7 +483,7 @@ posix_errno_t efile_open(const efile_path_t *path, enum efile_modes_t modes, /* Rewrite all failures on directories to EISDIR to match the old * driver. */ - if(has_file_attributes(path, FILE_ATTRIBUTE_DIRECTORY)) { + if(!(modes & EFILE_MODE_DIRECTORY) && has_file_attributes(path, FILE_ATTRIBUTE_DIRECTORY)) { return EISDIR; } @@ -751,6 +773,71 @@ static int is_name_surrogate(const efile_path_t *path) { return result; } +static void build_file_info(BY_HANDLE_FILE_INFORMATION *native_file_info, const efile_path_t *path, int is_link, efile_fileinfo_t *result) { + DWORD attributes; + + attributes = native_file_info->dwFileAttributes; + + if(is_link) { + result->type = EFILE_FILETYPE_SYMLINK; + /* This should be _S_IFLNK, but the old driver always set + * non-directories to _S_IFREG. */ + result->mode |= _S_IFREG; + } else if(attributes & FILE_ATTRIBUTE_DIRECTORY) { + result->type = EFILE_FILETYPE_DIRECTORY; + result->mode |= _S_IFDIR | _S_IEXEC; + } else { + if(is_executable_file(path)) { + result->mode |= _S_IEXEC; + } + + result->type = EFILE_FILETYPE_REGULAR; + result->mode |= _S_IFREG; + } + + if(!(attributes & FILE_ATTRIBUTE_READONLY)) { + result->access = EFILE_ACCESS_READ | EFILE_ACCESS_WRITE; + result->mode |= _S_IREAD | _S_IWRITE; + } else { + result->access = EFILE_ACCESS_READ; + result->mode |= _S_IREAD; + } + + /* Propagate user mode-bits to group/other fields */ + result->mode |= (result->mode & 0700) >> 3; + result->mode |= (result->mode & 0700) >> 6; + + result->size = + ((Uint64)native_file_info->nFileSizeHigh << 32ull) | + (Uint64)native_file_info->nFileSizeLow; + result->links = MAX(1, native_file_info->nNumberOfLinks); + + result->major_device = get_drive_number(path); + result->minor_device = 0; + result->inode = 0; + result->uid = 0; + result->gid = 0; +} + +static void build_file_info_times(BY_HANDLE_FILE_INFORMATION *native_file_info, efile_fileinfo_t *result) { + FILETIME_TO_EPOCH(result->m_time, native_file_info->ftLastWriteTime); + FILETIME_TO_EPOCH(result->a_time, native_file_info->ftLastAccessTime); + FILETIME_TO_EPOCH(result->c_time, native_file_info->ftCreationTime); + + if(result->m_time == -EPOCH_DIFFERENCE) { + /* Default to 1970 just like the old driver. */ + result->m_time = 0; + } + + if(result->a_time == -EPOCH_DIFFERENCE) { + result->a_time = result->m_time; + } + + if(result->c_time == -EPOCH_DIFFERENCE) { + result->c_time = result->m_time; + } +} + posix_errno_t efile_read_info(const efile_path_t *path, int follow_links, efile_fileinfo_t *result) { BY_HANDLE_FILE_INFORMATION native_file_info; DWORD attributes; @@ -770,23 +857,41 @@ posix_errno_t efile_read_info(const efile_path_t *path, int follow_links, efile_ return windows_to_posix_errno(last_error); } - attributes = FILE_ATTRIBUTE_DIRECTORY; + native_file_info.dwFileAttributes = FILE_ATTRIBUTE_DIRECTORY; } else if(is_path_root(path)) { /* Local (or mounted) roots can be queried with GetFileAttributesW but * lack support for GetFileInformationByHandle, so we'll skip that * part. */ + native_file_info.dwFileAttributes = attributes; } else { HANDLE handle; + DWORD last_error; + DWORD flags; if(attributes & FILE_ATTRIBUTE_REPARSE_POINT) { is_link = is_name_surrogate(path); } + flags = FILE_FLAG_BACKUP_SEMANTICS; + if(!follow_links && is_link) { + flags |= FILE_FLAG_OPEN_REPARSE_POINT; + } + + handle = CreateFileW((const WCHAR*)path->data, GENERIC_READ, + FILE_SHARE_FLAGS, NULL, OPEN_EXISTING, flags, NULL); + last_error = GetLastError(); + + if(handle == INVALID_HANDLE_VALUE) { + return windows_to_posix_errno(last_error); + } + if(follow_links && is_link) { posix_errno_t posix_errno; efile_path_t resolved_path; - posix_errno = internal_read_link(path, &resolved_path); + posix_errno = internal_read_link(handle, &resolved_path); + + CloseHandle(handle); if(posix_errno != 0) { return posix_errno; @@ -795,74 +900,43 @@ posix_errno_t efile_read_info(const efile_path_t *path, int follow_links, efile_ return efile_read_info(&resolved_path, 0, result); } - handle = CreateFileW((const WCHAR*)path->data, GENERIC_READ, - FILE_SHARE_FLAGS, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, - NULL); - - /* The old driver never cared whether this succeeded. */ - if(handle != INVALID_HANDLE_VALUE) { - GetFileInformationByHandle(handle, &native_file_info); - CloseHandle(handle); - } + GetFileInformationByHandle(handle, &native_file_info); + CloseHandle(handle); - FILETIME_TO_EPOCH(result->m_time, native_file_info.ftLastWriteTime); - FILETIME_TO_EPOCH(result->a_time, native_file_info.ftLastAccessTime); - FILETIME_TO_EPOCH(result->c_time, native_file_info.ftCreationTime); + build_file_info_times(&native_file_info, result); + } - if(result->m_time == -EPOCH_DIFFERENCE) { - /* Default to 1970 just like the old driver. */ - result->m_time = 0; - } + build_file_info(&native_file_info, path, is_link, result); - if(result->a_time == -EPOCH_DIFFERENCE) { - result->a_time = result->m_time; - } + return 0; +} - if(result->c_time == -EPOCH_DIFFERENCE) { - result->c_time = result->m_time; - } - } +posix_errno_t efile_read_handle_info(efile_data_t *d, efile_fileinfo_t *result) { + efile_win_t *w = (efile_win_t*)d; + HANDLE handle; + BY_HANDLE_FILE_INFORMATION native_file_info; + posix_errno_t posix_errno; + efile_path_t path; + int length; - if(is_link) { - result->type = EFILE_FILETYPE_SYMLINK; - /* This should be _S_IFLNK, but the old driver always set - * non-directories to _S_IFREG. */ - result->mode |= _S_IFREG; - } else if(attributes & FILE_ATTRIBUTE_DIRECTORY) { - result->type = EFILE_FILETYPE_DIRECTORY; - result->mode |= _S_IFDIR | _S_IEXEC; - } else { - if(is_executable_file(path)) { - result->mode |= _S_IEXEC; - } + sys_memset(&native_file_info, 0, sizeof(native_file_info)); - result->type = EFILE_FILETYPE_REGULAR; - result->mode |= _S_IFREG; + posix_errno = internal_read_link(w->handle, &path); + if(posix_errno != 0) { + return posix_errno; } - if(!(attributes & FILE_ATTRIBUTE_READONLY)) { - result->access = EFILE_ACCESS_READ | EFILE_ACCESS_WRITE; - result->mode |= _S_IREAD | _S_IWRITE; + if(GetFileInformationByHandle(w->handle, &native_file_info)) { + build_file_info_times(&native_file_info, result); + } else if(is_path_root(&path)) { + /* GetFileInformationByHandle is not supported on path roots, so + * fall back to efile_read_info. */ + return efile_read_info(&path, 0, result); } else { - result->access = EFILE_ACCESS_READ; - result->mode |= _S_IREAD; + return windows_to_posix_errno(GetLastError()); } - /* Propagate user mode-bits to group/other fields */ - result->mode |= (result->mode & 0700) >> 3; - result->mode |= (result->mode & 0700) >> 6; - - result->size = - ((Uint64)native_file_info.nFileSizeHigh << 32ull) | - (Uint64)native_file_info.nFileSizeLow; - - result->links = MAX(1, native_file_info.nNumberOfLinks); - - result->major_device = get_drive_number(path); - result->minor_device = 0; - result->inode = 0; - result->uid = 0; - result->gid = 0; + build_file_info(&native_file_info, &path, 0, result); return 0; } @@ -941,31 +1015,20 @@ posix_errno_t efile_set_time(const efile_path_t *path, Sint64 a_time, Sint64 m_t return windows_to_posix_errno(last_error); } -static posix_errno_t internal_read_link(const efile_path_t *path, efile_path_t *result) { +static posix_errno_t internal_read_link(HANDLE link_handle, efile_path_t *result) { DWORD required_length, actual_length; - HANDLE link_handle; DWORD last_error; - link_handle = CreateFileW((WCHAR*)path->data, GENERIC_READ, - FILE_SHARE_FLAGS, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); - last_error = GetLastError(); - - if(link_handle == INVALID_HANDLE_VALUE) { - return windows_to_posix_errno(last_error); - } - required_length = GetFinalPathNameByHandleW(link_handle, NULL, 0, 0); last_error = GetLastError(); if(required_length <= 0) { - CloseHandle(link_handle); return windows_to_posix_errno(last_error); } /* Unlike many other path functions (eg. GetFullPathNameW), this one * includes the NUL terminator in its required length. */ if(!enif_alloc_binary(required_length * sizeof(WCHAR), result)) { - CloseHandle(link_handle); return ENOMEM; } @@ -973,8 +1036,6 @@ static posix_errno_t internal_read_link(const efile_path_t *path, efile_path_t * (WCHAR*)result->data, required_length, 0); last_error = GetLastError(); - CloseHandle(link_handle); - if(actual_length == 0 || actual_length >= required_length) { enif_release_binary(result); return windows_to_posix_errno(last_error); @@ -992,6 +1053,8 @@ posix_errno_t efile_read_link(ErlNifEnv *env, const efile_path_t *path, ERL_NIF_ posix_errno_t posix_errno; ErlNifBinary result_bin; DWORD attributes; + HANDLE handle; + DWORD last_error; ASSERT_PATH_FORMAT(path); @@ -1007,7 +1070,17 @@ posix_errno_t efile_read_link(ErlNifEnv *env, const efile_path_t *path, ERL_NIF_ return EINVAL; } - posix_errno = internal_read_link(path, &result_bin); + handle = CreateFileW((WCHAR*)path->data, GENERIC_READ, + FILE_SHARE_FLAGS, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, + NULL); + last_error = GetLastError(); + + if(handle == INVALID_HANDLE_VALUE) { + return windows_to_posix_errno(last_error); + } + posix_errno = internal_read_link(handle, &result_bin); + + CloseHandle(handle); if(posix_errno == 0) { if(!normalize_path_result(&result_bin)) { diff --git a/erts/emulator/sys/common/erl_osenv.h b/erts/emulator/sys/common/erl_osenv.h index 4777f2148a..f2e96a6af7 100644 --- a/erts/emulator/sys/common/erl_osenv.h +++ b/erts/emulator/sys/common/erl_osenv.h @@ -35,16 +35,10 @@ # include "config.h" #endif -typedef struct __erts_osenv_data_t erts_osenv_data_t; - -typedef struct __erts_osenv_t { - struct __env_rbtnode_t *tree; - int variable_count; - int content_size; -} erts_osenv_t; - #include "sys.h" +typedef struct __erts_osenv_data_t erts_osenv_data_t; + struct __erts_osenv_data_t { Sint length; void *data; diff --git a/erts/emulator/sys/unix/erl_child_setup.c b/erts/emulator/sys/unix/erl_child_setup.c index 129861ebd5..8f261761db 100644 --- a/erts/emulator/sys/unix/erl_child_setup.c +++ b/erts/emulator/sys/unix/erl_child_setup.c @@ -63,10 +63,13 @@ #include "erl_driver.h" #include "sys_uds.h" -#include "hash.h" #include "erl_term.h" #include "erl_child_setup.h" +#undef ERTS_GLB_INLINE_INCL_FUNC_DEF +#define ERTS_GLB_INLINE_INCL_FUNC_DEF 1 +#include "hash.h" + #define SET_CLOEXEC(fd) fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC) #if defined(__ANDROID__) @@ -75,6 +78,10 @@ #define SHELL "/bin/sh" #endif /* __ANDROID__ */ +#if !defined(MSG_DONTWAIT) && defined(MSG_NONBLOCK) +#define MSG_DONTWAIT MSG_NONBLOCK +#endif + //#define HARD_DEBUG #ifdef HARD_DEBUG #define DEBUG_PRINT(fmt, ...) fprintf(stderr, "%d:" fmt "\r\n", getpid(), ##__VA_ARGS__) diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index 4823e549ea..78866b356c 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -740,10 +740,17 @@ void os_version(int *pMajor, int *pMinor, int *pBuild) { * X.Y or X.Y.Z. */ (void) uname(&uts); +#ifdef _AIX + /* AIX stores the major in version and minor in release */ + *pMajor = atoi(uts.version); + *pMinor = atoi(uts.release); + *pBuild = 0; /* XXX: get oslevel for AIX or TR on i */ +#else release = uts.release; *pMajor = get_number(&release); /* Pointer to major version. */ *pMinor = get_number(&release); /* Pointer to minor version. */ *pBuild = get_number(&release); /* Pointer to build number. */ +#endif } void erts_do_break_handling(void) diff --git a/erts/emulator/sys/unix/sys_drivers.c b/erts/emulator/sys/unix/sys_drivers.c index 92020c6f35..870d0bd28a 100644 --- a/erts/emulator/sys/unix/sys_drivers.c +++ b/erts/emulator/sys/unix/sys_drivers.c @@ -55,6 +55,7 @@ #define WANT_NONBLOCKING /* must define this to pull in defs from sys.h */ #include "sys.h" +#include "erl_osenv.h" #include "erl_threads.h" diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c index b95aadc9b2..bc3de42be7 100644 --- a/erts/emulator/sys/win32/sys.c +++ b/erts/emulator/sys/win32/sys.c @@ -27,6 +27,7 @@ #endif #include "sys.h" +#include "erl_osenv.h" #include "erl_alloc.h" #include "erl_sys_driver.h" #include "global.h" diff --git a/erts/emulator/sys/win32/sys_env.c b/erts/emulator/sys/win32/sys_env.c index c78161b344..36223c2c14 100644 --- a/erts/emulator/sys/win32/sys_env.c +++ b/erts/emulator/sys/win32/sys_env.c @@ -23,6 +23,7 @@ #endif #include "sys.h" +#include "erl_osenv.h" #include "erl_sys_driver.h" #include "erl_alloc.h" diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile index 019af2162f..731aa66924 100644 --- a/erts/emulator/test/Makefile +++ b/erts/emulator/test/Makefile @@ -90,6 +90,7 @@ MODULES= \ gc_SUITE \ guard_SUITE \ hash_SUITE \ + hash_property_test_SUITE \ hibernate_SUITE \ hipe_SUITE \ iovec_SUITE \ @@ -252,7 +253,7 @@ release_tests_spec: make_emakefile $(INSTALL_DATA) $(NO_OPT_ERL_FILES) "$(RELSYSDIR)" $(INSTALL_DATA) $(NATIVE_ERL_FILES) "$(RELSYSDIR)" chmod -R u+w "$(RELSYSDIR)" - tar cf - *_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -) + tar cf - *_SUITE_data property_test | (cd "$(RELSYSDIR)"; tar xf -) release_docs_spec: diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl index c5abd04e07..cde2bbfa7c 100644 --- a/erts/emulator/test/bif_SUITE.erl +++ b/erts/emulator/test/bif_SUITE.erl @@ -29,8 +29,8 @@ shadow_comments/1,list_to_utf8_atom/1, specs/1,improper_bif_stubs/1,auto_imports/1, t_list_to_existing_atom/1,os_env/1,otp_7526/1, - binary_to_atom/1,binary_to_existing_atom/1, - atom_to_binary/1,min_max/1, erlang_halt/1, + t_binary_to_atom/1,t_binary_to_existing_atom/1, + t_atom_to_binary/1,min_max/1, erlang_halt/1, erl_crash_dump_bytes/1, is_builtin/1, error_stacktrace/1, error_stacktrace_during_call_trace/1, @@ -50,7 +50,7 @@ all() -> specs, improper_bif_stubs, auto_imports, t_list_to_existing_atom, os_env, otp_7526, display, display_string, list_to_utf8_atom, - atom_to_binary, binary_to_atom, binary_to_existing_atom, + t_atom_to_binary, t_binary_to_atom, t_binary_to_existing_atom, erl_crash_dump_bytes, min_max, erlang_halt, is_builtin, error_stacktrace, error_stacktrace_during_call_trace, group_leader_prio, group_leader_prio_dirty, @@ -496,7 +496,7 @@ test_7526(N) -> -define(BADARG(E), {'EXIT',{badarg,_}} = (catch E)). -define(SYS_LIMIT(E), {'EXIT',{system_limit,_}} = (catch E)). -binary_to_atom(Config) when is_list(Config) -> +t_binary_to_atom(Config) when is_list(Config) -> HalfLong = lists:seq(0, 127), HalfLongAtom = list_to_atom(HalfLong), HalfLongBin = list_to_binary(HalfLong), @@ -524,8 +524,10 @@ binary_to_atom(Config) when is_list(Config) -> test_binary_to_atom(<<C/utf8>>, utf8) end], - <<"こんにちは"/utf8>> = - atom_to_binary(test_binary_to_atom(<<"こんにちは"/utf8>>, utf8), utf8), + ExoticBin = <<"こんにちは"/utf8>>, + ExoticAtom = test_binary_to_atom(ExoticBin, utf8), + ExoticBin = atom_to_binary(ExoticAtom, utf8), + ExoticBin = atom_to_binary(ExoticAtom), %% badarg failures. fail_binary_to_atom(atom), @@ -543,6 +545,7 @@ binary_to_atom(Config) when is_list(Config) -> %% Bad UTF8 sequences. ?BADARG(binary_to_atom(id(<<255>>), utf8)), + ?BADARG(binary_to_atom(id(<<255>>))), ?BADARG(binary_to_atom(id(<<255,0>>), utf8)), ?BADARG(binary_to_atom(id(<<16#C0,16#80>>), utf8)), %Overlong 0. <<B:1/binary, _/binary>> = id(<<194, 163>>), %Truncated character ERL-474 @@ -550,6 +553,7 @@ binary_to_atom(Config) when is_list(Config) -> %% system_limit failures. ?SYS_LIMIT(binary_to_atom(id(<<0:512/unit:8,255>>), utf8)), + ?SYS_LIMIT(binary_to_atom(id(<<0:512/unit:8,255>>))), ?SYS_LIMIT(binary_to_atom(id(<<0:512/unit:8,255,0>>), utf8)), ?SYS_LIMIT(binary_to_atom(<<0:256/unit:8>>, latin1)), ?SYS_LIMIT(binary_to_atom(<<0:257/unit:8>>, latin1)), @@ -562,6 +566,14 @@ binary_to_atom(Config) when is_list(Config) -> test_binary_to_atom(Bin0, Encoding) -> Res = binary_to_atom(Bin0, Encoding), Res = binary_to_existing_atom(Bin0, Encoding), + if + Encoding =:= utf8; + Encoding =:= unicode -> + Res = binary_to_atom(Bin0), + Res = binary_to_existing_atom(Bin0); + true -> + ok + end, Bin1 = id(<<7:3,Bin0/binary,32:5>>), Sz = byte_size(Bin0), <<_:3,UnalignedBin:Sz/binary,_:5>> = Bin1, @@ -581,6 +593,12 @@ fail_binary_to_atom(Bin) -> ok end, try + binary_to_atom(Bin) + catch + error:badarg -> + ok + end, + try binary_to_existing_atom(Bin, latin1) catch error:badarg -> @@ -591,10 +609,16 @@ fail_binary_to_atom(Bin) -> catch error:badarg -> ok + end, + try + binary_to_existing_atom(Bin) + catch + error:badarg -> + ok end. -binary_to_existing_atom(Config) when is_list(Config) -> +t_binary_to_existing_atom(Config) when is_list(Config) -> UnlikelyBin = <<"ou0897979655678dsfj923874390867er869fds973qerueoru">>, try binary_to_existing_atom(UnlikelyBin, latin1), @@ -609,6 +633,12 @@ binary_to_existing_atom(Config) when is_list(Config) -> catch error:badarg -> ok end, + try + binary_to_existing_atom(UnlikelyBin), + ct:fail(atom_exists) + catch + error:badarg -> ok + end, UnlikelyAtom = binary_to_atom(id(UnlikelyBin), latin1), UnlikelyAtom = binary_to_existing_atom(UnlikelyBin, latin1), @@ -625,7 +655,7 @@ binary_to_existing_atom(Config) when is_list(Config) -> ok. -atom_to_binary(Config) when is_list(Config) -> +t_atom_to_binary(Config) when is_list(Config) -> HalfLong = lists:seq(0, 127), HalfLongAtom = list_to_atom(HalfLong), HalfLongBin = list_to_binary(HalfLong), @@ -641,12 +671,15 @@ atom_to_binary(Config) when is_list(Config) -> LongBin = atom_to_binary(LongAtom, latin1), %% utf8. + <<>> = atom_to_binary(''), <<>> = atom_to_binary('', utf8), <<>> = atom_to_binary('', unicode), <<127>> = atom_to_binary('\177', utf8), <<"abcdef">> = atom_to_binary(abcdef, utf8), HalfLongBin = atom_to_binary(HalfLongAtom, utf8), + HalfLongBin = atom_to_binary(HalfLongAtom), LongAtomBin = atom_to_binary(LongAtom, utf8), + LongAtomBin = atom_to_binary(LongAtom), verify_long_atom_bin(LongAtomBin, 0), %% Failing cases. @@ -678,8 +711,15 @@ fail_atom_to_binary(Term) -> catch error:badarg -> ok + end, + try + atom_to_binary(Term) + catch + error:badarg -> + ok end. + min_max(Config) when is_list(Config) -> a = erlang:min(id(a), a), a = erlang:min(id(a), b), @@ -860,6 +900,7 @@ error_stacktrace_test() -> Types = [apply_const_last, apply_const, apply_last, apply, double_apply_const_last, double_apply_const, double_apply_last, double_apply, multi_apply_const_last, + apply_const_only, apply_only, multi_apply_const, multi_apply_last, multi_apply, call_const_last, call_last, call_const, call], lists:foreach(fun (Type) -> @@ -897,6 +938,8 @@ error_stacktrace_test() -> ok. stk([], Type, Func) -> + put(erlang, erlang), + put(tail, []), tail(Type, Func, jump), ok; stk([_|L], Type, Func) -> @@ -910,6 +953,12 @@ tail(Type, error_1, do) -> tail(Type, error_2, do) -> do_error_2(Type). +do_error_2(apply_const_only) -> + apply(erlang, error, [oops, [apply_const_only]]); +do_error_2(apply_only) -> + Erlang = get(erlang), + Tail = get(tail), + apply(Erlang, error, [oops, [apply_only|Tail]]); do_error_2(apply_const_last) -> erlang:apply(erlang, error, [oops, [apply_const_last]]); do_error_2(apply_const) -> @@ -951,6 +1000,12 @@ do_error_2(call) -> erlang:error(id(oops), id([call])). +do_error_1(apply_const_only) -> + apply(erlang, error, [oops]); +do_error_1(apply_only) -> + Erlang = get(erlang), + Tail = get(tail), + apply(Erlang, error, [oops|Tail]); do_error_1(apply_const_last) -> erlang:apply(erlang, error, [oops]); do_error_1(apply_const) -> diff --git a/erts/emulator/test/call_trace_SUITE.erl b/erts/emulator/test/call_trace_SUITE.erl index 742592f88e..477b0f5bb3 100644 --- a/erts/emulator/test/call_trace_SUITE.erl +++ b/erts/emulator/test/call_trace_SUITE.erl @@ -832,21 +832,27 @@ deep_exception() -> R1 -> ct:fail({returned,abbr(R1)}) catch error:badarg -> ok end, - expect(fun ({trace,S,call,{lists,reverse,[L1,L2]}}) + expect(fun ({trace,S,call,{lists,reverse,[L1,L2]}}, Traps) when is_list(L1), is_list(L2), S == Self -> - next; + %% Each trapping call to reverse/2 must have a corresponding + %% exception_from + {next, Traps + 1}; ({trace,S,exception_from, - {lists,reverse,2},{error,badarg}}) + {lists,reverse,2},{error,badarg}}, Traps) + when S == Self, Traps > 1 -> + {next, Traps - 1}; + ({trace,S,exception_from, + {lists,reverse,2},{error,badarg}}, 1) when S == Self -> expected; - ('_') -> + ('_', _Traps) -> {trace,Self,exception_from, {lists,reverse,2},{error,badarg}}; - (_) -> + (_, _Traps) -> {unexpected, {trace,Self,exception_from, {lists,reverse,2},{error,badarg}}} - end), + end, 0), deep_exception(?LINE, deep_5, [1,2], 7, [{trace,Self,call,{erlang,error,[undef]}}, {trace,Self,exception_from,{erlang,error,1}, @@ -896,21 +902,27 @@ deep_exception() -> R2 -> ct:fail({returned,abbr(R2)}) catch error:badarg -> ok end, - expect(fun ({trace,S,call,{lists,reverse,[L1,L2]}}) + expect(fun ({trace,S,call,{lists,reverse,[L1,L2]}}, Traps) when is_list(L1), is_list(L2), S == Self -> - next; + %% Each trapping call to reverse/2 must have a corresponding + %% exception_from + {next, Traps + 1}; + ({trace,S,exception_from, + {lists,reverse,2},{error,badarg}}, Traps) + when S == Self, Traps > 1 -> + {next, Traps - 1}; ({trace,S,exception_from, - {lists,reverse,2},{error,badarg}}) + {lists,reverse,2},{error,badarg}}, 1) when S == Self -> expected; - ('_') -> + ('_', _Traps) -> {trace,Self,exception_from, {lists,reverse,2},{error,badarg}}; - (_) -> + (_, _Traps) -> {unexpected, {trace,Self,exception_from, {lists,reverse,2},{error,badarg}}} - end), + end, 0), deep_exception(?LINE, apply, [?MODULE,deep_5,[1,2]], 7, [{trace,Self,call,{erlang,error,[undef]}}, {trace,Self,exception_from,{erlang,error,1}, @@ -975,21 +987,27 @@ deep_exception() -> R3 -> ct:fail({returned,abbr(R3)}) catch error:badarg -> ok end, - expect(fun ({trace,S,call,{lists,reverse,[L1,L2]}}) + expect(fun ({trace,S,call,{lists,reverse,[L1,L2]}}, Traps) when is_list(L1), is_list(L2), S == Self -> - next; + %% Each trapping call to reverse/2 must have a corresponding + %% exception_from + {next, Traps + 1}; + ({trace,S,exception_from, + {lists,reverse,2},{error,badarg}}, Traps) + when S == Self, Traps > 1 -> + {next, Traps - 1}; ({trace,S,exception_from, - {lists,reverse,2},{error,badarg}}) + {lists,reverse,2},{error,badarg}}, 1) when S == Self -> expected; - ('_') -> + ('_', _Traps) -> {trace,Self,exception_from, {lists,reverse,2},{error,badarg}}; - (_) -> + (_, _Traps) -> {unexpected, {trace,Self,exception_from, {lists,reverse,2},{error,badarg}}} - end), + end, 0), deep_exception(?LINE, apply, [fun () -> ?MODULE:deep_5(1,2) end, []], 7, [{trace,Self,call,{erlang,error,[undef]}}, @@ -1249,6 +1267,24 @@ expect(Message) -> ct:fail(no_trace_message) end. +expect(Validator, State0) when is_function(Validator) -> + receive + M -> + case Validator(M, State0) of + expected -> + ok = io:format("Expected and got ~p", [abbr(M)]); + {next, State} -> + ok = io:format("Expected and got ~p", [abbr(M)]), + expect(Validator, State); + {unexpected,Message} -> + io:format("Expected ~p; got ~p", [abbr(Message),abbr(M)]), + ct:fail({unexpected,abbr([M|flush()])}) + end + after 5000 -> + io:format("Expected ~p; got nothing", [abbr(Validator('_'))]), + ct:fail(no_trace_message) + end. + trace_info(What, Key) -> get(tracer) ! {apply,self(),{erlang,trace_info,[What,Key]}}, Res = receive diff --git a/erts/emulator/test/dirty_bif_SUITE.erl b/erts/emulator/test/dirty_bif_SUITE.erl index 4f5ad0295a..2ded862b8a 100644 --- a/erts/emulator/test/dirty_bif_SUITE.erl +++ b/erts/emulator/test/dirty_bif_SUITE.erl @@ -397,7 +397,9 @@ dirty_process_trace(Config) when is_list(Config) -> access_dirty_process( Config, fun() -> - erlang:trace_pattern({erts_debug,dirty_io,2}, + %% BIFs can only be traced when their modules are loaded. + code:ensure_loaded(erts_debug), + 1 = erlang:trace_pattern({erts_debug,dirty_io,2}, [{'_',[],[{return_trace}]}], [local,meta]), ok diff --git a/erts/emulator/test/emulator.spec b/erts/emulator/test/emulator.spec index 7a6dd83020..087bd8880d 100644 --- a/erts/emulator/test/emulator.spec +++ b/erts/emulator/test/emulator.spec @@ -1,2 +1,3 @@ {enable_builtin_hooks, false}. {suites,"../emulator_test",all}. +{skip_groups,"../emulator_test",hash_SUITE,[phash2_benchmark],"Benchmark only"}. diff --git a/erts/emulator/test/emulator_bench.spec b/erts/emulator/test/emulator_bench.spec index 03638bfa23..8b1bb71a40 100644 --- a/erts/emulator/test/emulator_bench.spec +++ b/erts/emulator/test/emulator_bench.spec @@ -1,3 +1,4 @@ {groups,"../emulator_test",estone_SUITE,[estone_bench]}. {groups,"../emulator_test",binary_SUITE,[iolist_size_benchmarks]}. {groups,"../emulator_test",erts_debug_SUITE,[interpreter_size_bench]}. +{groups,"../emulator_test",hash_SUITE,[phash2_benchmark]}. diff --git a/erts/emulator/test/hash_SUITE.erl b/erts/emulator/test/hash_SUITE.erl index 3cbb3c7d5f..dd71c3da58 100644 --- a/erts/emulator/test/hash_SUITE.erl +++ b/erts/emulator/test/hash_SUITE.erl @@ -33,7 +33,25 @@ -module(hash_SUITE). -export([basic_test/0,cmp_test/1,range_test/0,spread_test/1, phash2_test/0, otp_5292_test/0, - otp_7127_test/0]). + otp_7127_test/0, + run_phash2_benchmarks/0, + test_phash2_binary_aligned_and_unaligned_equal/1, + test_phash2_4GB_plus_bin/1, + test_phash2_10MB_plus_bin/1, + test_phash2_large_map/1, + test_phash2_shallow_long_list/1, + test_phash2_deep_list/1, + test_phash2_deep_tuple/1, + test_phash2_deep_tiny/1, + test_phash2_with_42/1, + test_phash2_with_short_tuple/1, + test_phash2_with_short_list/1, + test_phash2_with_tiny_bin/1, + test_phash2_with_tiny_unaligned_sub_binary/1, + test_phash2_with_small_unaligned_sub_binary/1, + test_phash2_with_large_bin/1, + test_phash2_with_large_unaligned_sub_binary/1, + test_phash2_with_super_large_unaligned_sub_binary/1]). %% %% Define to run outside of test server @@ -43,13 +61,15 @@ %% %% Define for debug output %% -%-define(debug,1). +-define(debug,1). -ifdef(STANDALONE). -define(config(A,B),config(A,B)). +-record(event, {name, data}). -export([config/2]). -else. -include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). -endif. -ifdef(debug). @@ -67,12 +87,15 @@ -ifdef(STANDALONE). config(priv_dir,_) -> ".". +notify(X) -> + erlang:display(X). -else. %% When run in test server. --export([all/0, suite/0, +-export([groups/0, all/0, suite/0, test_basic/1,test_cmp/1,test_range/1,test_spread/1, test_phash2/1,otp_5292/1,bit_level_binaries/1,otp_7127/1, - test_hash_zero/1]). + test_hash_zero/1, init_per_suite/1, end_per_suite/1, + init_per_group/2, end_per_group/2]). suite() -> [{ct_hooks,[ts_install_cth]}, @@ -81,7 +104,71 @@ suite() -> all() -> [test_basic, test_cmp, test_range, test_spread, test_phash2, otp_5292, bit_level_binaries, otp_7127, - test_hash_zero]. + test_hash_zero, test_phash2_binary_aligned_and_unaligned_equal, + test_phash2_4GB_plus_bin, + test_phash2_10MB_plus_bin, + {group, phash2_benchmark_tests}, + {group, phash2_benchmark}]. + +get_phash2_benchmarks() -> + [ + test_phash2_large_map, + test_phash2_shallow_long_list, + test_phash2_deep_list, + test_phash2_deep_tuple, + test_phash2_deep_tiny, + test_phash2_with_42, + test_phash2_with_short_tuple, + test_phash2_with_short_list, + test_phash2_with_tiny_bin, + test_phash2_with_tiny_unaligned_sub_binary, + test_phash2_with_small_unaligned_sub_binary, + test_phash2_with_large_bin, + test_phash2_with_large_unaligned_sub_binary, + test_phash2_with_super_large_unaligned_sub_binary + ]. + +groups() -> + [ + { + phash2_benchmark_tests, + [], + get_phash2_benchmarks() + }, + { + phash2_benchmark, + [], + get_phash2_benchmarks() + } + ]. + + +init_per_suite(Config) -> + io:format("START APPS~n"), + A0 = case application:start(sasl) of + ok -> [sasl]; + _ -> [] + end, + A = case application:start(os_mon) of + ok -> [os_mon|A0]; + _ -> A0 + end, + io:format("APPS STARTED~n"), + [{started_apps, A}|Config]. + +end_per_suite(Config) -> + As = proplists:get_value(started_apps, Config), + lists:foreach(fun (A) -> application:stop(A) end, As), + Config. + +init_per_group(phash2_benchmark_tests, Config) -> + [phash2_benchmark_tests |Config]; +init_per_group(_, Config) -> + Config. + +end_per_group(_, Config) -> + Config. + %% Tests basic functionality of erlang:phash and that the %% hashes has not changed (neither hash nor phash) @@ -119,6 +206,9 @@ otp_7127(Config) when is_list(Config) -> test_hash_zero(Config) when is_list(Config) -> hash_zero_test(). + +notify(X) -> + ct_event:notify(X). -endif. @@ -133,26 +223,17 @@ basic_test() -> 16#77777777777777],16#FFFFFFFF), ExternalReference = <<131,114,0,3,100,0,13,110,111,110,111,100,101,64, 110,111,104,111,115,116,0,0,0,0,122,0,0,0,0,0,0,0,0>>, - 1113403635 = erlang:phash(binary_to_term(ExternalReference), - 16#FFFFFFFF), - ExternalFun = <<131,117,0,0,0,3,103,100,0,13,110,111,110,111,100,101,64, - 110,111,104,111,115,116,0,0,0,38,0,0,0,0,0,100,0,8,101, - 114,108,95,101,118,97,108,97,20,98,5,182,139,98,108,0,0, - 0,3,104,2,100,0,1,66,109,0,0,0,33,131,114,0,3,100,0,13, - 110,111,110,111,100,101,64,110,111,104,111,115,116,0,0, - 0,0,122,0,0,0,0,0,0,0,0,104,2,100,0,1,76,107,0,33,131, - 114,0,3,100,0,13,110,111,110,111,100,101,64,110,111,104, - 111,115,116,0,0,0,0,122,0,0,0,0,0,0,0,0,104,2,100,0,1,82, - 114,0,3,100,0,13,110,111,110,111,100,101,64,110,111,104, - 111,115,116,0,0,0,0,122,0,0,0,0,0,0,0,0,106,108,0,0,0,1, - 104,5,100,0,6,99,108,97,117,115,101,97,1,106,106,108,0,0, - 0,1,104,3,100,0,7,105,110,116,101,103,101,114,97,1,97,1, - 106,106,104,3,100,0,4,101,118,97,108,104,2,100,0,5,115, - 104,101,108,108,100,0,10,108,111,99,97,108,95,102,117, - 110,99,108,0,0,0,1,103,100,0,13,110,111,110,111,100,101, - 64,110,111,104,111,115,116,0,0,0,22,0,0,0,0,0,106>>, - 170987488 = erlang:phash(binary_to_term(ExternalFun), - 16#FFFFFFFF), + ExternalReference = <<131,114,0,3,100,0,13,110,111,110,111,100,101,64, + 110,111,104,111,115,116,0,0,0,0,122,0,0,0,0,0,0,0,0>>, + 1113403635 = phash_from_external(ExternalReference), + + ExternalFun = <<131,112,0,0,0,70,1,212,190,220,28,179,144,194,131, + 19,215,105,97,77,251,125,93,0,0,0,0,0,0,0,2,100,0,1, + 116,97,0,98,6,165,246,224,103,100,0,13,110,111, + 110,111,100,101,64,110,111,104,111,115,116,0,0,0,91, + 0,0,0,0,0,97,2,97,1>>, + 25769064 = phash_from_external(ExternalFun), + case (catch erlang:phash(1,0)) of {'EXIT',{badarg, _}} -> ok; @@ -160,6 +241,8 @@ basic_test() -> exit(phash_accepted_zero_as_range) end. +phash_from_external(Ext) -> + erlang:phash(binary_to_term(Ext), 16#FFFFFFFF). range_test() -> F = fun(From,From,_FF) -> @@ -354,6 +437,7 @@ phash2_test() -> %% bit-level binaries {<<0:7>>, 1055790816}, + {(fun()-> B = <<255,7:3>>, <<_:4,D/bitstring>> = B, D end)(), 911751529}, {<<"abc",13:4>>, 670412287}, {<<5:3,"12345678901234567890">>, 289973273}, @@ -424,6 +508,159 @@ phash2_test() -> [] = [{E,H,H2} || {E,H} <- L, (H2 = erlang:phash2(E, Max)) =/= H], ok. +test_phash2_binary_aligned_and_unaligned_equal(Config) when is_list(Config) -> + erts_debug:set_internal_state(available_internal_state, true), + test_aligned_and_unaligned_equal_up_to(256*12+255), + erts_debug:set_internal_state(available_internal_state, false). + +test_aligned_and_unaligned_equal_up_to(BinSize) -> + Results = + lists:map(fun(Size) -> + test_aligned_and_unaligned_equal(Size) + end, lists:seq(1, BinSize)), + %% DataDir = filename:join(filename:dirname(code:which(?MODULE)), "hash_SUITE_data"), + %% ExpResFile = filename:join(DataDir, "phash2_bin_expected_results.txt"), + %% {ok, [ExpRes]} = file:consult(ExpResFile), + %% %% ok = file:write_file(ExpResFile, io_lib:format("~w.~n", [Results])), + %% Results = ExpRes, + 110469206 = erlang:phash2(Results). + +test_aligned_and_unaligned_equal(BinSize) -> + Bin = make_random_bin(BinSize), + LastByte = last_byte(Bin), + LastInBitstring = LastByte rem 11, + Bitstring = << Bin/binary, <<LastInBitstring:5>>/bitstring >>, + UnalignedBin = make_unaligned_sub_bitstring(Bin), + UnalignedBitstring = make_unaligned_sub_bitstring(Bitstring), + case erts_debug:get_internal_state(available_internal_state) of + false -> erts_debug:set_internal_state(available_internal_state, true); + _ -> ok + end, + erts_debug:set_internal_state(reds_left, 3), + BinHash = erlang:phash2(Bin), + BinHash = erlang:phash2(Bin), + erts_debug:set_internal_state(reds_left, 3), + UnalignedBinHash = erlang:phash2(UnalignedBin), + UnalignedBinHash = erlang:phash2(UnalignedBin), + BinHash = UnalignedBinHash, + erts_debug:set_internal_state(reds_left, 3), + BitstringHash = erlang:phash2(Bitstring), + BitstringHash = erlang:phash2(Bitstring), + erts_debug:set_internal_state(reds_left, 3), + UnalignedBitstringHash = erlang:phash2(UnalignedBitstring), + UnalignedBitstringHash = erlang:phash2(UnalignedBitstring), + BitstringHash = UnalignedBitstringHash, + {BinHash, BitstringHash}. + +last_byte(Bin) -> + NotLastByteSize = (erlang:bit_size(Bin)) - 8, + <<_:NotLastByteSize/bitstring, LastByte:8>> = Bin, + LastByte. + +test_phash2_4GB_plus_bin(Config) when is_list(Config) -> + run_when_enough_resources( + fun() -> + erts_debug:set_internal_state(available_internal_state, true), + %% Created Bin4GB here so it only needs to be created once + erts_debug:set_internal_state(force_gc, self()), + Bin4GB = get_4GB_bin(), + test_phash2_plus_bin_helper1(Bin4GB, <<>>, <<>>, 13708901), + erts_debug:set_internal_state(force_gc, self()), + test_phash2_plus_bin_helper1(Bin4GB, <<>>, <<3:5>>, 66617678), + erts_debug:set_internal_state(force_gc, self()), + test_phash2_plus_bin_helper1(Bin4GB, <<13>>, <<>>, 31308392), + erts_debug:set_internal_state(force_gc, self()), + erts_debug:set_internal_state(available_internal_state, false) + end). + + +test_phash2_10MB_plus_bin(Config) when is_list(Config) -> + erts_debug:set_internal_state(available_internal_state, true), + erts_debug:set_internal_state(force_gc, self()), + Bin10MB = get_10MB_bin(), + test_phash2_plus_bin_helper1(Bin10MB, <<>>, <<>>, 22776267), + erts_debug:set_internal_state(force_gc, self()), + test_phash2_plus_bin_helper1(Bin10MB, <<>>, <<3:5>>, 124488972), + erts_debug:set_internal_state(force_gc, self()), + test_phash2_plus_bin_helper1(Bin10MB, <<13>>, <<>>, 72958346), + erts_debug:set_internal_state(force_gc, self()), + erts_debug:set_internal_state(available_internal_state, false). + +get_10MB_bin() -> + TmpBin = make_random_bin(10239), + Bin = erlang:iolist_to_binary([0, TmpBin]), + IOList10MB = duplicate_iolist(Bin, 10), + Bin10MB = erlang:iolist_to_binary(IOList10MB), + 10485760 = size(Bin10MB), + Bin10MB. + +get_4GB_bin() -> + TmpBin = make_random_bin(65535), + Bin = erlang:iolist_to_binary([0, TmpBin]), + IOList4GB = duplicate_iolist(Bin, 16), + Bin4GB = erlang:iolist_to_binary(IOList4GB), + 4294967296 = size(Bin4GB), + Bin4GB. + +duplicate_iolist(IOList, 0) -> + IOList; +duplicate_iolist(IOList, NrOfTimes) -> + duplicate_iolist([IOList, IOList], NrOfTimes - 1). + +test_phash2_plus_bin_helper1(Bin4GB, ExtraBytes, ExtraBits, ExpectedHash) -> + test_phash2_plus_bin_helper2(Bin4GB, fun id/1, ExtraBytes, ExtraBits, ExpectedHash), + test_phash2_plus_bin_helper2(Bin4GB, fun make_unaligned_sub_bitstring/1, ExtraBytes, ExtraBits, ExpectedHash). + +test_phash2_plus_bin_helper2(Bin, TransformerFun, ExtraBytes, ExtraBits, ExpectedHash) -> + ExtraBitstring = << ExtraBytes/binary, ExtraBits/bitstring >>, + LargerBitstring = << ExtraBytes/binary, + ExtraBits/bitstring, + Bin/bitstring >>, + LargerTransformedBitstring = TransformerFun(LargerBitstring), + ExtraBitstringHash = erlang:phash2(ExtraBitstring), + ExpectedHash = + case size(LargerTransformedBitstring) < 4294967296 of + true -> + erts_debug:set_internal_state(force_gc, self()), + erts_debug:set_internal_state(reds_left, 1), + Hash = erlang:phash2(LargerTransformedBitstring), + Hash = erlang:phash2(LargerTransformedBitstring), + Hash; + false -> + erts_debug:set_internal_state(force_gc, self()), + erts_debug:set_internal_state(reds_left, 1), + ExtraBitstringHash = erlang:phash2(LargerTransformedBitstring), + ExtraBitstringHash = erlang:phash2(LargerTransformedBitstring), + ExtraBitstringHash + end. + +run_when_enough_resources(Fun) -> + case {total_memory(), erlang:system_info(wordsize)} of + {Mem, 8} when is_integer(Mem) andalso Mem >= 31 -> + Fun(); + {Mem, WordSize} -> + {skipped, + io_lib:format("Not enough resources (System Memory >= ~p, Word Size = ~p)", + [Mem, WordSize])} + end. + +%% Total memory in GB +total_memory() -> + try + MemoryData = memsup:get_system_memory_data(), + case lists:keysearch(total_memory, 1, MemoryData) of + {value, {total_memory, TM}} -> + TM div (1024*1024*1024); + false -> + {value, {system_total_memory, STM}} = + lists:keysearch(system_total_memory, 1, MemoryData), + STM div (1024*1024*1024) + end + catch + _ : _ -> + undefined + end. + -ifdef(FALSE). f1() -> abc. @@ -436,14 +673,23 @@ f3(X, Y) -> -endif. otp_5292_test() -> - PH = fun(E) -> [erlang:phash(E, 1 bsl 32), - erlang:phash(-E, 1 bsl 32), - erlang:phash2(E, 1 bsl 32), - erlang:phash2(-E, 1 bsl 32)] - end, + PH = fun(E) -> + EInList = [1, 2, 3, E], + EInList2 = [E, 1, 2, 3], + NegEInList = [1, 2, 3, -E], + NegEInList2 = [-E, 1, 2, 3], + [erlang:phash(E, 1 bsl 32), + erlang:phash(-E, 1 bsl 32), + erlang:phash2(E, 1 bsl 32), + erlang:phash2(-E, 1 bsl 32), + erlang:phash2(EInList, 1 bsl 32), + erlang:phash2(EInList2, 1 bsl 32), + erlang:phash2(NegEInList, 1 bsl 32), + erlang:phash2(NegEInList2, 1 bsl 32)] + end, S2 = md5([md5(hash_int(S, E, PH)) || {Start, N, Sz} <- d(), {S, E} <- int(Start, N, Sz)]), - <<124,81,198,121,174,233,19,137,10,83,33,80,226,111,238,99>> = S2, + <<234,63,192,76,253,57,250,32,44,11,73,1,161,102,14,238>> = S2, ok. d() -> @@ -684,3 +930,313 @@ unaligned_sub_bitstr(Bin0) when is_bitstring(Bin0) -> id(I) -> I. + +%% Benchmarks for phash2 + +run_phash2_benchmarks() -> + Benchmarks = [ + test_phash2_large_map, + test_phash2_shallow_long_list, + test_phash2_deep_list, + test_phash2_deep_tuple, + test_phash2_deep_tiny, + test_phash2_with_42, + test_phash2_with_short_tuple, + test_phash2_with_short_list, + test_phash2_with_tiny_bin, + test_phash2_with_tiny_unaligned_sub_binary, + test_phash2_with_small_unaligned_sub_binary, + test_phash2_with_large_bin, + test_phash2_with_large_unaligned_sub_binary, + test_phash2_with_super_large_unaligned_sub_binary + ], + [print_comment(B) || B <- Benchmarks]. + + +print_comment(FunctionName) -> + io:format("~p~n", [FunctionName]), + io:format("~s~n", [element(2, erlang:apply(?MODULE, FunctionName, [[]]))]). + +nr_of_iters(BenchmarkNumberOfIterations, Config) -> + case lists:member(phash2_benchmark_tests, Config) of + true -> 1; + false -> BenchmarkNumberOfIterations + end. + + +test_phash2_large_map(Config) when is_list(Config) -> + {Size, ExpectedHash} = + case {total_memory(), erlang:system_info(wordsize)} of + {Mem, 8} when is_integer(Mem) andalso Mem > 2 -> + {1000000, 121857429}; + _ -> + {1000, 66609305} + end, + run_phash2_test_and_benchmark(nr_of_iters(45, Config), + get_map(Size), + ExpectedHash). + +test_phash2_shallow_long_list(Config) when is_list(Config) -> + {Size, ExpectedHash} = + case {total_memory(), erlang:system_info(wordsize)} of + {Mem, 8} when is_integer(Mem) andalso Mem > 2 -> + {1000000, 78700388}; + _ -> + {1000, 54749638} + end, + run_phash2_test_and_benchmark(nr_of_iters(1, Config), + lists:duplicate(Size, get_complex_tuple()), + ExpectedHash). + +test_phash2_deep_list(Config) when is_list(Config) -> + {Size, ExpectedHash} = + case {total_memory(), erlang:system_info(wordsize)} of + {Mem, 8} when is_integer(Mem) andalso Mem > 2 -> + {500000, 17986444}; + _ -> + {1000, 81794308} + end, + run_phash2_test_and_benchmark(nr_of_iters(1, Config), + make_deep_list(Size, get_complex_tuple()), + ExpectedHash). + +test_phash2_deep_tuple(Config) when is_list(Config) -> + {Size, ExpectedHash} = + case {total_memory(), erlang:system_info(wordsize)} of + {Mem, 8} when is_integer(Mem) andalso Mem > 2 -> + {500000, 116594715}; + _ -> + {500, 109057352} + end, + run_phash2_test_and_benchmark(nr_of_iters(1, Config), + make_deep_tuple(Size, get_complex_tuple()), + ExpectedHash). + +test_phash2_deep_tiny(Config) when is_list(Config) -> + run_phash2_test_and_benchmark(nr_of_iters(1000000, Config), + make_deep_list(19, 42), + 111589624). + +test_phash2_with_42(Config) when is_list(Config) -> + run_phash2_test_and_benchmark(nr_of_iters(20000000, Config), + 42, + 30328728). + +test_phash2_with_short_tuple(Config) when is_list(Config) -> + run_phash2_test_and_benchmark(nr_of_iters(10000000, Config), + {a,b,<<"hej">>, "hej"}, + 50727199). + +test_phash2_with_short_list(Config) when is_list(Config) -> + run_phash2_test_and_benchmark(nr_of_iters(10000000, Config), + [a,b,"hej", "hello"], + 117108642). + +test_phash2_with_tiny_bin(Config) when is_list(Config) -> + run_phash2_test_and_benchmark(nr_of_iters(20000000, Config), + make_random_bin(10), + 129616602). + +test_phash2_with_tiny_unaligned_sub_binary(Config) when is_list(Config) -> + run_phash2_test_and_benchmark(nr_of_iters(10000000, Config), + make_unaligned_sub_binary(make_random_bin(11)), + 59364725). + +test_phash2_with_small_unaligned_sub_binary(Config) when is_list(Config) -> + run_phash2_test_and_benchmark(nr_of_iters(400000, Config), + make_unaligned_sub_binary(make_random_bin(1001)), + 130388119). + +test_phash2_with_large_bin(Config) when is_list(Config) -> + {Size, ExpectedHash} = + case {total_memory(), erlang:system_info(wordsize)} of + {Mem, 8} when is_integer(Mem) andalso Mem > 2 -> + {10000000, 48249379}; + _ -> + {1042, 14679520} + end, + run_phash2_test_and_benchmark(nr_of_iters(150, Config), + make_random_bin(Size), + ExpectedHash). + +test_phash2_with_large_unaligned_sub_binary(Config) when is_list(Config) -> + {Size, ExpectedHash} = + case {total_memory(), erlang:system_info(wordsize)} of + {Mem, 8} when is_integer(Mem) andalso Mem > 2 -> + {10000001, 122836437}; + _ -> + {10042, 127144287} + end, + run_phash2_test_and_benchmark(nr_of_iters(50, Config), + make_unaligned_sub_binary(make_random_bin(Size)), + ExpectedHash). + +test_phash2_with_super_large_unaligned_sub_binary(Config) when is_list(Config) -> + {Size, ExpectedHash} = + case {total_memory(), erlang:system_info(wordsize)} of + {Mem, 8} when is_integer(Mem) andalso Mem > 2 -> + {20000001, 112086727}; + _ -> + {20042, 91996619} + end, + run_phash2_test_and_benchmark(nr_of_iters(20, Config), + make_unaligned_sub_binary(make_random_bin(Size)), + ExpectedHash). + +make_deep_list(1, Item) -> + {Item, Item}; +make_deep_list(Depth, Item) -> + [{Item, Item}, make_deep_list(Depth - 1, Item)]. + +make_deep_tuple(1, Item) -> + [Item, Item]; +make_deep_tuple(Depth, Item) -> + {[Item, Item], make_deep_tuple(Depth - 1, Item)}. + +% Helper functions for benchmarking + +loop(0, _) -> ok; +loop(Iterations, Fun) -> + Fun(), + loop(Iterations - 1, Fun). + +run_phash2_test_and_benchmark(Iterations, Term, ExpectedHash) -> + Parent = self(), + Test = + fun() -> + Hash = erlang:phash2(Term), + case ExpectedHash =:= Hash of + false -> + Parent ! {got_bad_hash, Hash}, + ExpectedHash = Hash; + _ -> ok + end + end, + Benchmark = + fun() -> + garbage_collect(), + {Time, _} =timer:tc(fun() -> loop(Iterations, Test) end), + Parent ! Time + end, + spawn(Benchmark), + receive + {got_bad_hash, Hash} -> + ExpectedHash = Hash; + Time -> + TimeInS = case (Time/1000000) of + 0.0 -> 0.0000000001; + T -> T + end, + IterationsPerSecond = Iterations / TimeInS, + notify(#event{ name = benchmark_data, data = [{value, IterationsPerSecond}]}), + {comment, io_lib:format("Iterations per second: ~p, Iterations ~p, Benchmark time: ~p seconds)", + [IterationsPerSecond, Iterations, Time/1000000])} + end. + +get_complex_tuple() -> + BPort = <<131,102,100,0,13,110,111,110,111,100,101,64,110,111,104, + 111,115,116,0,0,0,1,0>>, + Port = binary_to_term(BPort), + + BXPort = <<131,102,100,0,11,97,112,97,64,108,101,103,111,108,97,115, + 0,0,0,24,3>>, + XPort = binary_to_term(BXPort), + + BRef = <<131,114,0,3,100,0,13,110,111,110,111,100,101,64,110,111,104, + 111,115,116,0,0,0,1,255,0,0,0,0,0,0,0,0>>, + Ref = binary_to_term(BRef), + + BXRef = <<131,114,0,3,100,0,11,97,112,97,64,108,101,103,111,108,97,115, + 2,0,0,0,155,0,0,0,0,0,0,0,0>>, + XRef = binary_to_term(BXRef), + + BXPid = <<131,103,100,0,11,97,112,97,64,108,101,103,111,108,97,115, + 0,0,0,36,0,0,0,0,1>>, + XPid = binary_to_term(BXPid), + + + %% X = f1(), Y = f2(), Z = f3(X, Y), + + %% F1 = fun f1/0, % -> abc + B1 = <<131,112,0,0,0,66,0,215,206,77,69,249,50,170,17,129,47,21,98, + 13,196,76,242,0,0,0,1,0,0,0,0,100,0,1,116,97,1,98,2,195,126, + 58,103,100,0,13,110,111,110,111,100,101,64,110,111,104,111, + 115,116,0,0,0,112,0,0,0,0,0>>, + F1 = binary_to_term(B1), + + %% F2 = fun f2/0, % -> abd + B2 = <<131,112,0,0,0,66,0,215,206,77,69,249,50,170,17,129,47,21,98, + 13,196,76,242,0,0,0,2,0,0,0,0,100,0,1,116,97,2,98,3,130,152, + 185,103,100,0,13,110,111,110,111,100,101,64,110,111,104,111, + 115,116,0,0,0,112,0,0,0,0,0>>, + F2 = binary_to_term(B2), + + %% F3 = fun f3/2, % -> {abc, abd} + B3 = <<131,112,0,0,0,66,2,215,206,77,69,249,50,170,17,129,47,21,98, + 13,196,76,242,0,0,0,3,0,0,0,0,100,0,1,116,97,3,98,7,168,160, + 93,103,100,0,13,110,111,110,111,100,101,64,110,111,104,111, + 115,116,0,0,0,112,0,0,0,0,0>>, + F3 = binary_to_term(B3), + + %% F4 = fun () -> 123456789012345678901234567 end, + B4 = <<131,112,0,0,0,66,0,215,206,77,69,249,50,170,17,129,47,21,98, + 13,196,76,242,0,0,0,4,0,0,0,0,100,0,1,116,97,4,98,2,230,21, + 171,103,100,0,13,110,111,110,111,100,101,64,110,111,104,111, + 115,116,0,0,0,112,0,0,0,0,0>>, + F4 = binary_to_term(B4), + + %% F5 = fun() -> {X,Y,Z} end, + B5 = <<131,112,0,0,0,92,0,215,206,77,69,249,50,170,17,129,47,21,98, + 13,196,76,242,0,0,0,5,0,0,0,3,100,0,1,116,97,5,98,0,99,101, + 130,103,100,0,13,110,111,110,111,100,101,64,110,111,104,111, + 115,116,0,0,0,112,0,0,0,0,0,100,0,3,97,98,99,100,0,3,97,98, + 100,104,2,100,0,3,97,98,99,100,0,3,97,98,100>>, + F5 = binary_to_term(B5), + {{1,{2}},an_atom, 1, 3434.923942394,<<"this is a binary">>, + make_unaligned_sub_binary(<<"this is also a binary">>),c,d,e,f,g,h,i,j,k,l,[f], + 999999999999999999666666662123123123123324234999999999999999, 234234234, + BPort, Port, BXPort, XPort, BRef, Ref, BXRef, XRef, BXPid, XPid, F1, F2, F3, F4, F5, + #{a => 1, b => 2, c => 3, d => 4, e => 5, f => 6, g => 7, h => 8, i => 9, + j => 1, k => 1, l => 123123123123213, m => [1,2,3,4,5,6,7,8], o => 5, p => 6, + q => 7, r => 8, s => 9}}. + +get_map_helper(MapSoFar, 0) -> + MapSoFar; +get_map_helper(MapSoFar, NumOfItemsToAdd) -> + NewMapSoFar = maps:put(NumOfItemsToAdd, NumOfItemsToAdd, MapSoFar), + get_map_helper(NewMapSoFar, NumOfItemsToAdd -1). + +get_map(Size) -> + get_map_helper(#{}, Size). + + +%% Copied from binary_SUITE +make_unaligned_sub_binary(Bin0) when is_binary(Bin0) -> + Bin1 = <<0:3,Bin0/binary,31:5>>, + Sz = size(Bin0), + <<0:3,Bin:Sz/binary,31:5>> = id(Bin1), + Bin. + +make_unaligned_sub_bitstring(Bin0) -> + Bin1 = <<0:3,Bin0/bitstring,31:5>>, + Sz = erlang:bit_size(Bin0), + <<0:3,Bin:Sz/bitstring,31:5>> = id(Bin1), + Bin. + +make_random_bin(Size) -> + make_random_bin(Size, []). + +make_random_bin(0, Acc) -> + iolist_to_binary(Acc); +make_random_bin(Size, []) -> + make_random_bin(Size - 1, [simple_rand() rem 256]); +make_random_bin(Size, [N | Tail]) -> + make_random_bin(Size - 1, [simple_rand(N) rem 256, N |Tail]). + +simple_rand() -> + 123456789. +simple_rand(Seed) -> + A = 1103515245, + C = 12345, + M = (1 bsl 31), + (A * Seed + C) rem M. diff --git a/erts/emulator/test/hash_property_test_SUITE.erl b/erts/emulator/test/hash_property_test_SUITE.erl new file mode 100644 index 0000000000..b4c7810a52 --- /dev/null +++ b/erts/emulator/test/hash_property_test_SUITE.erl @@ -0,0 +1,103 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2004-2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +%% + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% %%% +%%% WARNING %%% +%%% %%% +%%% This is experimental code which may be changed or removed %%% +%%% anytime without any warning. %%% +%%% %%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +-module(hash_property_test_SUITE). + +-export([suite/0,all/0,groups/0,init_per_suite/1, + end_per_suite/1,init_per_group/2,end_per_group/2]). + +-export([test_phash2_no_diff/1, + test_phash2_no_diff_long/1, + test_phash2_no_diff_between_versions/1]). + +-include_lib("common_test/include/ct.hrl"). + +suite() -> + [{ct_hooks,[ts_install_cth]}]. + +all() -> [{group, proper}]. + +groups() -> + [{proper, [], [test_phash2_no_diff, + test_phash2_no_diff_long, + test_phash2_no_diff_between_versions]}]. + + +%%% First prepare Config and compile the property tests for the found tool: +init_per_suite(Config) -> + ct_property_test:init_per_suite(Config). + +end_per_suite(Config) -> + Config. + +%%% Only proper is supported +init_per_group(proper, Config) -> + case proplists:get_value(property_test_tool,Config) of + proper -> Config; + X -> {skip, lists:concat([X," is not supported"])} + end; +init_per_group(_, Config) -> + Config. + +end_per_group(_, Config) -> + Config. + +test_phash2_no_diff(Config) when is_list(Config) -> + true = ct_property_test:quickcheck( + phash2_properties:prop_phash2_same_with_same_input(), + Config). + +test_phash2_no_diff_long(Config) when is_list(Config) -> + true = ct_property_test:quickcheck( + phash2_properties:prop_phash2_same_with_same_long_input(), + Config). + +test_phash2_no_diff_between_versions(Config) when is_list(Config) -> + R = "21", + case test_server:is_release_available(R) of + true -> + Rel = {release,R}, + case test_server:start_node(rel21,peer,[{erl,[Rel]}]) of + {error, Reason} -> {skip, io_lib:format("Could not start node: ~p~n", [Reason])}; + {ok, Node} -> + try + true = ct_property_test:quickcheck( + phash2_properties:prop_phash2_same_in_different_versions(Node), + Config), + true = ct_property_test:quickcheck( + phash2_properties:prop_phash2_same_in_different_versions_with_long_input(Node), + Config) + after + test_server:stop_node(Node) + end + end; + false -> + {skip, io_lib:format("Release ~s not available~n", [R])} + end. diff --git a/erts/emulator/test/hibernate_SUITE.erl b/erts/emulator/test/hibernate_SUITE.erl index a20f306e04..d65d0ff2fd 100644 --- a/erts/emulator/test/hibernate_SUITE.erl +++ b/erts/emulator/test/hibernate_SUITE.erl @@ -46,12 +46,17 @@ all() -> basic(Config) when is_list(Config) -> Ref = make_ref(), Info = {self(),Ref}, - ExpectedHeapSz = erts_debug:size([Info]), + ExpectedHeapSz = expected_heap_size([Info]), Child = spawn_link(fun() -> basic_hibernator(Info) end), hibernate_wake_up(100, ExpectedHeapSz, Child), Child ! please_quit_now, ok. +expected_heap_size(Term) -> + %% When hibernating, an extra word will be allocated on the stack + %% for a continuation pointer. + erts_debug:size(Term) + 1. + hibernate_wake_up(0, _, _) -> ok; hibernate_wake_up(N, ExpectedHeapSz, Child) -> {heap_size,Before} = process_info(Child, heap_size), @@ -142,7 +147,7 @@ whats_up_calc(A1, A2, A3, A4, A5, A6, A7, A8, A9, Acc) -> dynamic_call(Config) when is_list(Config) -> Ref = make_ref(), Info = {self(),Ref}, - ExpectedHeapSz = erts_debug:size([Info]), + ExpectedHeapSz = expected_heap_size([Info]), Child = spawn_link(fun() -> ?MODULE:dynamic_call_hibernator(Info, hibernate) end), hibernate_wake_up(100, ExpectedHeapSz, Child), Child ! please_quit_now, diff --git a/erts/emulator/test/match_spec_SUITE.erl b/erts/emulator/test/match_spec_SUITE.erl index 21de6b1002..686b431876 100644 --- a/erts/emulator/test/match_spec_SUITE.erl +++ b/erts/emulator/test/match_spec_SUITE.erl @@ -198,7 +198,8 @@ caller_and_return_to(Config) -> {trace,Tracee,call,{?MODULE,do_the_put,[test]},{?MODULE,do_put,1}}, {trace,Tracee,call,{erlang,integer_to_list,[1]},{?MODULE,do_the_put,1}}, {trace,Tracee,return_to,{?MODULE,do_the_put,1}}, - {trace,Tracee,call,{erlang,put,[test,"1"]},{?MODULE,do_put,1}}, + {trace,Tracee,call,{erlang,put,[test,"1"]},{?MODULE,do_the_put,1}}, + {trace,Tracee,return_to,{?MODULE,do_the_put,1}}, {trace,Tracee,return_to,{?MODULE,do_put,1}}, %% These last trace messages are a bit strange... diff --git a/erts/emulator/test/nofrag_SUITE.erl b/erts/emulator/test/nofrag_SUITE.erl index 8b1519ae36..d4c74579e2 100644 --- a/erts/emulator/test/nofrag_SUITE.erl +++ b/erts/emulator/test/nofrag_SUITE.erl @@ -22,6 +22,11 @@ -include_lib("common_test/include/ct.hrl"). +%% This suite alters the return values of functions which breaks certain +%% assumptions made by the compiler, so we have to turn off module-level type +%% optimization to be safe. +-compile(no_module_opt). + -export([all/0, suite/0, error_handler/1,error_handler_apply/1, error_handler_fixed_apply/1,error_handler_fun/1, diff --git a/erts/emulator/test/property_test/phash2_properties.erl b/erts/emulator/test/property_test/phash2_properties.erl new file mode 100644 index 0000000000..b1f3207c56 --- /dev/null +++ b/erts/emulator/test/property_test/phash2_properties.erl @@ -0,0 +1,63 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2019-2019. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%% + +-module(phash2_properties). + +-ifdef(PROPER). + +-include_lib("proper/include/proper.hrl"). +-export([prop_phash2_same_with_same_input/0, + prop_phash2_same_with_same_long_input/0, + prop_phash2_same_in_different_versions/1, + prop_phash2_same_in_different_versions_with_long_input/1]). +-proptest([proper]). + +%%-------------------------------------------------------------------- +%% Properties -------------------------------------------------------- +%%-------------------------------------------------------------------- + +prop_phash2_same_with_same_input() -> + ?FORALL(T, any(), erlang:phash2(T) =:= erlang:phash2(T)). + +prop_phash2_same_with_same_long_input() -> + ?FORALL(T, any(), + begin + BigTerm = lists:duplicate(10000, T), + erlang:phash2(BigTerm) =:= erlang:phash2(BigTerm) + end). + +prop_phash2_same_in_different_versions(DifferntVersionNode) -> + ?FORALL(T, any(), + erlang:phash2(T) =:= rpc:call(DifferntVersionNode,erlang,phash2,[T])). + +prop_phash2_same_in_different_versions_with_long_input(DifferntVersionNode) -> + ?FORALL(T, any(), + begin + BigTerm = lists:duplicate(10000, T), + RpcRes = rpc:call(DifferntVersionNode,erlang,phash2,[BigTerm]), + LocalRes = erlang:phash2(BigTerm), + RpcRes =:= LocalRes + end). + +%%-------------------------------------------------------------------- +%% Generators ------------------------------------------------------- +%%-------------------------------------------------------------------- + +-endif. diff --git a/erts/emulator/test/trace_call_time_SUITE.erl b/erts/emulator/test/trace_call_time_SUITE.erl index 26f96a1766..9bab4cbbd8 100644 --- a/erts/emulator/test/trace_call_time_SUITE.erl +++ b/erts/emulator/test/trace_call_time_SUITE.erl @@ -254,8 +254,7 @@ combo(Config) when is_list(Config) -> 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, [], [local]), 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, true, [call_time]), 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, MetaMs, [{meta,MetaTracer}]), - %% not implemented - %2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, true, [call_count]), + 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, true, [call_count]), 1 = erlang:trace(Self, true, [{tracer,LocalTracer} | Flags]), %% @@ -284,9 +283,7 @@ combo(Config) when is_list(Config) -> {value,{match_spec,[]}} = lists:keysearch(match_spec, 1, TraceInfoBif), {value,{meta, MetaTracer}} = lists:keysearch(meta, 1, TraceInfoBif), {value,{meta_match_spec,MetaMs}} = lists:keysearch(meta_match_spec, 1, TraceInfoBif), - %% not implemented - {value,{call_count,false}} = lists:keysearch(call_count, 1, TraceInfoBif), - %{value,{call_count,0}} = lists:keysearch(call_count, 1, TraceInfoBif), + {value,{call_count,0}} = lists:keysearch(call_count, 1, TraceInfoBif), {value,{call_time,[]}} = lists:keysearch(call_time, 1, TraceInfoBif), %% @@ -429,6 +426,8 @@ called_function(Config) when is_list(Config) -> %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% dead_tracer(Config) when is_list(Config) -> + TracedMFAs = dead_tracer_mfas(), + Self = self(), FirstTracer = tracer(), StartTracing = fun() -> turn_on_tracing(Self) end, @@ -443,14 +442,14 @@ dead_tracer(Config) when is_list(Config) -> erlang:yield(), %% Collect and check that we only get call_time info for the current process. - Info1 = collect_all_info(), + Info1 = collect_all_info(TracedMFAs), [] = other_than_self(Info1), io:format("~p\n", [Info1]), %% Note that we have not turned off tracing for the current process, %% but that the tracer has terminated. No more call_time information should be recorded. [1,2,3] = seq(1, 3, fun(I) -> I + 1 end), - [] = collect_all_info(), + [] = collect_all_info(TracedMFAs), %% When we start a second tracer process, that tracer process must %% not inherit the tracing flags and the dead tracer (even though @@ -459,7 +458,7 @@ dead_tracer(Config) when is_list(Config) -> tell_tracer(SecondTracer, StartTracing), Seq20 = lists:seq(1, 20), Seq20 = seq(1, 20, fun(I) -> I + 1 end), - Info2 = collect_all_info(), + Info2 = collect_all_info(TracedMFAs), io:format("~p\n", [Info2]), [] = other_than_self(Info2), SecondTracer ! quit, @@ -495,9 +494,21 @@ turn_on_tracing(Pid) -> _ = now(), ok. -collect_all_info() -> - collect_all_info([{?MODULE,F,A} || {F,A} <- module_info(functions)] ++ - erlang:system_info(snifs)). +%% We want to trace functions local to this module as well as all BIFs, and for +%% the latter we need to ensure that their modules are loaded. +dead_tracer_mfas() -> + Modules = [M || {M,_F,_A} <- erlang:system_info(snifs)], + Whitelist0 = gb_sets:from_list(Modules), + Whitelist = case code:ensure_modules_loaded(Modules) of + {error, Reasons} -> + Blacklist = gb_sets:from_list([M || {M, _} <- Reasons]), + gb_sets:subtract(Whitelist0, Blacklist); + ok -> + Whitelist0 + end, + EligibleSNIFs = [MFA || {M,_F,_A}=MFA <- erlang:system_info(snifs), + gb_sets:is_element(M, Whitelist)], + [{?MODULE,F,A} || {F,A} <- module_info(functions)] ++ EligibleSNIFs. collect_all_info([MFA|T]) -> CallTime = erlang:trace_info(MFA, call_time), @@ -567,21 +578,29 @@ seq_r(Start, Stop, Succ, R) -> seq_r(Succ(Start), Stop, Succ, [Start | R]). % Check call time tracing data and print mismatches -check_trace_info(Mfa, [{Pid, C,_,_}] = Expect, Time) -> - case erlang:trace_info(Mfa, call_time) of - % Time tests are somewhat problematic. We want to know if Time (EXPECTED_TIME) and S*1000000 + Us (ACTUAL_TIME) - % is the same. - % If the ratio EXPECTED_TIME/ACTUAL_TIME is ~ 1 or if EXPECTED_TIME - ACTUAL_TIME is near zero, the test is ok. - {call_time,[{Pid,C,S,Us}]} when S >= 0, Us >= 0, abs(1 - Time/(S*1000000 + Us)) < ?R_ERROR; abs(Time - S*1000000 - Us) < ?US_ERROR -> +check_trace_info(Mfa, [{Pid, ExpectedC,_,_}] = Expect, Time) -> + {call_time,[{Pid,C,S,Us}]} = erlang:trace_info(Mfa, call_time), + {Mod, Name, Arity} = Mfa, + IsBuiltin = erlang:is_builtin(Mod, Name, Arity), + if + %% Call count on BIFs may exceed number of calls as they often trap to + %% themselves. + IsBuiltin, C >= ExpectedC, S >= 0, Us >= 0, + abs(1 - Time/(S*1000000 + Us)) < ?R_ERROR; + abs(Time - S*1000000 - Us) < ?US_ERROR -> ok; - {call_time,[{Pid,C,S,Us}]} -> + not IsBuiltin, C =:= ExpectedC, S >= 0, Us >= 0, + abs(1 - Time/(S*1000000 + Us)) < ?R_ERROR; + abs(Time - S*1000000 - Us) < ?US_ERROR -> + ok; + true -> Sum = S*1000000 + Us, - io:format("Expected ~p -> {call_time, ~p (Time ~p us)}~n - got ~w s. ~w us. = ~w us. - ~w -> delta ~w (ratio ~.2f, should be 1.0)~n", - [Mfa, Expect, Time, S, Us, Sum, Time, Sum - Time, Time/Sum]), - time_error; - Other -> - io:format("Expected ~p -> {call_time, ~p (Time ~p us)}~n - got ~p~n", [ Mfa, Expect, Time, Other]), - time_count_error + io:format("Expected ~p -> {call_time, ~p (Time ~p us)}~n - got ~w " + "s. ~w us. = ~w us. - ~w -> delta ~w (ratio ~.2f, " + "should be 1.0)~n", + [Mfa, Expect, Time, + S, Us, Sum, Time, Sum - Time, Time/Sum]), + time_error end; check_trace_info(Mfa, Expect, _) -> case erlang:trace_info(Mfa, call_time) of @@ -670,9 +689,12 @@ loop() -> quit -> ok; {Pid, execute, Fun } when is_function(Fun) -> + %% Make sure we always run with the same amount of reductions. + erlang:yield(), Pid ! {self(), answer, erlang:apply(Fun, [])}, loop(); {Pid, execute, {M, F, A}} -> + erlang:yield(), Pid ! {self(), answer, erlang:apply(M, F, A)}, loop() end. diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops index 605a402f2a..cc21f9b5b4 100755 --- a/erts/emulator/utils/beam_makeops +++ b/erts/emulator/utils/beam_makeops @@ -1315,7 +1315,7 @@ sub combine_instruction_group { my $inc = 0; unless ($i == $#slots) { - $flags = "-no_next"; + $flags = "-micro_instruction"; my $next_offset = $label_to_offset{$next}; $inc = ($offset + $size) - $next_offset; $transfer_to_next = "I += $inc;\n" if $inc; @@ -1553,8 +1553,10 @@ sub code_gen { my $dispatch_next; my $instr_offset = $group_size + $offset + 1; - if ($flags =~ /-no_next/) { + if ($flags =~ /-micro_instruction/) { $dispatch_next = ""; + } elsif ($flags =~ /-no_next/) { + $dispatch_next = "ASSERT(!\"Fell through '$name' (-no_next)\");"; } elsif ($flags =~ /-no_prefetch/) { $dispatch_next = "\nI += $instr_offset;\n" . "ASSERT(VALID_INSTR(*I));\n" . diff --git a/erts/emulator/utils/make_tables b/erts/emulator/utils/make_tables index deee5c2344..ef0df28fc8 100755 --- a/erts/emulator/utils/make_tables +++ b/erts/emulator/utils/make_tables @@ -35,7 +35,6 @@ use File::Basename; # Output: # <-src>/erl_am.c # <-src>/erl_bif_table.c -# <-src>/erl_bif_wrap.c # <-src>/erl_dirty_bif_wrap.c # <-src>/erl_guard_bifs.c # <-src>/hipe_nbif_impl.c @@ -92,7 +91,7 @@ while (<>) { my($type, @args) = split; if ($type eq 'atom') { save_atoms(@args); - } elsif ($type eq 'bif' or $type eq 'ubif' or $type eq 'gcbif') { + } elsif ($type eq 'bif' or $type eq 'ubif' or $type eq 'hbif') { if (@args > 2) { error("$type only allows two arguments"); } @@ -124,14 +123,22 @@ while (<>) { error("invalid sched_type: $sched_type"); } - my $wrapper; - if ($type eq 'bif') { - $wrapper = "wrap_$alias"; - } else { - $wrapper = $alias; - } + my $kind; + if ($type eq 'bif') { + $kind = 'BIF_KIND_REGULAR'; + } + elsif ($type eq 'hbif') { + $kind = 'BIF_KIND_HEAVY'; + } + elsif ($type eq 'ubif') { + $kind = 'BIF_KIND_GUARD'; + } + else { + error("invalid bif_type: $type"); + } + push(@bif, ["am_$atom_alias{$mod}","am_$atom_alias{$name}",$arity, - $alias3,$wrapper,$alias]); + $alias3,$alias,$kind]); push(@bif_info, [$type, $sched_type, $alias3, $alias]); } elsif ($type eq 'dirty-cpu' or $type eq 'dirty-io' or $type eq 'dirty-cpu-test' or $type eq 'dirty-io-test') { @@ -196,7 +203,7 @@ open_file("$include/erl_bif_list.h"); my $i; for ($i = 0; $i < @bif; $i++) { # module atom, function atom, arity, C function, table index - print "BIF_LIST($bif[$i]->[0],$bif[$i]->[1],$bif[$i]->[2],$bif[$i]->[3],$bif[$i]->[5],$i)\n"; + print "BIF_LIST($bif[$i]->[0],$bif[$i]->[1],$bif[$i]->[2],$bif[$i]->[3],$bif[$i]->[4],$i)\n"; } # @@ -208,15 +215,24 @@ my $bif_size = @bif; print <<EOF; #ifndef __ERL_BIF_TABLE_H__ #define __ERL_BIF_TABLE_H__ + +#include "sys.h" + typedef void *BifFunction; +typedef enum { + BIF_KIND_REGULAR, + BIF_KIND_HEAVY, + BIF_KIND_GUARD +} BifKind; + typedef struct bif_entry { Eterm module; Eterm name; int arity; BifFunction f; - BifFunction traced; BifFunction impl; + BifKind kind; } BifEntry; typedef struct erts_gc_bif { @@ -232,7 +248,6 @@ typedef struct erts_u_bif { extern BifEntry bif_table[]; extern Export* bif_export[]; -extern const ErtsGcBif erts_gc_bifs[]; extern const ErtsUBif erts_u_bifs[]; #define BIF_SIZE $bif_size @@ -250,7 +265,6 @@ for ($i = 0; $i < @bif; $i++) { my $args = join(', ', 'Process*', 'Eterm*', 'UWord*'); my $name = $bif_info[$i]->[3]; print "Eterm $name($args);\n"; - print "Eterm wrap_$name($args);\n"; print "Eterm erts_gc_$name(Process* p, Eterm* reg, Uint live);\n" if $bif_info[$i]->[0] eq 'gcbif'; print "Eterm $bif_info[$i]->[2]($args);\n" @@ -283,25 +297,6 @@ for ($i = 0; $i < @bif; $i++) { print "};\n\n"; # -# Generate the bif wrappers file. -# - -open_file("$src/erl_bif_wrap.c"); -my $i; -includes("export.h", "sys.h", "erl_vm.h", "global.h", "erl_process.h", "bif.h", - "erl_bif_table.h", "erl_atom_table.h"); -for ($i = 0; $i < @bif; $i++) { - next if $bif[$i]->[3] eq $bif[$i]->[4]; # Skip unwrapped bifs - my $arity = $bif[$i]->[2]; - my $func = $bif_info[$i]->[3]; - print "Eterm\n"; - print "wrap_$func(Process* p, Eterm* args, UWord* I)\n"; - print "{\n"; - print " return erts_bif_trace($i, p, args, I);\n"; - print "}\n\n"; -} - -# # Generate erl_gc_bifs.c. # @@ -309,18 +304,11 @@ open_file("$src/erl_guard_bifs.c"); my $i; includes("export.h", "sys.h", "erl_vm.h", "global.h", "erl_process.h", "bif.h", "erl_bif_table.h"); -print "const ErtsGcBif erts_gc_bifs[] = {\n"; -for ($i = 0; $i < @bif; $i++) { - next unless $bif_info[$i]->[0] eq 'gcbif'; - print " {$bif[$i]->[3], erts_gc_$bif[$i]->[3], BIF_$bif[$i]->[5]},\n"; -} -print " {NULL, NULL, -1}\n"; -print "};\n"; print "const ErtsUBif erts_u_bifs[] = {\n"; for ($i = 0; $i < @bif; $i++) { next unless $bif_info[$i]->[0] eq 'ubif'; - print " {$bif[$i]->[3], BIF_$bif[$i]->[5]},\n"; + print " {$bif[$i]->[3], BIF_$bif[$i]->[4]},\n"; } print " {NULL, -1}\n"; print "};\n"; @@ -368,7 +356,7 @@ EOF my $i; for ($i = 0; $i < @bif; $i++) { print <<EOF; -Eterm nbif_impl_$bif[$i]->[5](Process *c_p, Eterm *regs); +Eterm nbif_impl_$bif[$i]->[4](Process *c_p, Eterm *regs); EOF } @@ -388,9 +376,9 @@ EOF for ($i = 0; $i < @bif; $i++) { print <<EOF; -Eterm nbif_impl_$bif[$i]->[5](Process *c_p, Eterm *regs) +Eterm nbif_impl_$bif[$i]->[4](Process *c_p, Eterm *regs) { - return $bif[$i]->[3](c_p, regs, (UWord *) bif_export\[BIF_$bif[$i]->[5]\]); + return $bif[$i]->[3](c_p, regs, (UWord *) bif_export\[BIF_$bif[$i]->[4]\]); } EOF diff --git a/erts/epmd/Makefile b/erts/epmd/Makefile index d3308ddedc..e4b201bd88 100644 --- a/erts/epmd/Makefile +++ b/erts/epmd/Makefile @@ -31,3 +31,5 @@ SPECIAL_TARGETS = # Default Subdir Targets # ---------------------------------------------------- include $(ERL_TOP)/make/otp_subdir.mk + +include $(ERL_TOP)/make/app_targets.mk diff --git a/erts/epmd/epmd.mk b/erts/epmd/epmd.mk index b1fd04dc04..f6889a7ff1 100644 --- a/erts/epmd/epmd.mk +++ b/erts/epmd/epmd.mk @@ -67,5 +67,5 @@ EPMD_NODE_TYPE=110 # Distribution format 5 contains the new md5 based handshake. EPMD_DIST_LOW=5 -EPMD_DIST_HIGH=5 +EPMD_DIST_HIGH=6 diff --git a/erts/epmd/src/epmd.h b/erts/epmd/src/epmd.h index cffcd4ae7a..7332294d3d 100644 --- a/erts/epmd/src/epmd.h +++ b/erts/epmd/src/epmd.h @@ -26,6 +26,7 @@ #define EPMD_ALIVE2_REQ 'x' #define EPMD_PORT2_REQ 'z' #define EPMD_ALIVE2_RESP 'y' +#define EPMD_ALIVE2_X_RESP 'v' #define EPMD_PORT2_RESP 'w' #define EPMD_NAMES_REQ 'n' diff --git a/erts/epmd/src/epmd_int.h b/erts/epmd/src/epmd_int.h index ed9bbdb8cd..a5156a142e 100644 --- a/erts/epmd/src/epmd_int.h +++ b/erts/epmd/src/epmd_int.h @@ -277,6 +277,12 @@ static const struct in6_addr in6addr_loopback = #define put_int16(i, s) {((unsigned char*)(s))[0] = ((i) >> 8) & 0xff; \ ((unsigned char*)(s))[1] = (i) & 0xff;} +#define put_int32(i, s) do {((char*)(s))[0] = (char)((i) >> 24) & 0xff; \ + ((char*)(s))[1] = (char)((i) >> 16) & 0xff; \ + ((char*)(s))[2] = (char)((i) >> 8) & 0xff; \ + ((char*)(s))[3] = (char)(i) & 0xff;} \ + while (0) + #if defined(__GNUC__) # define EPMD_INLINE __inline__ #elif defined(__WIN32__) @@ -307,10 +313,10 @@ struct enode { int fd; /* The socket in use */ unsigned short port; /* Port number of Erlang node */ char symname[MAXSYMLEN+1]; /* Name of the Erlang node */ - short creation; /* Started as a random number 1..3 */ + unsigned int cr_counter; /* Used to generate 'creation' numbers */ char nodetype; /* 77 = normal erlang node 72 = hidden (c-node */ char protocol; /* 0 = tcp/ipv4 */ - unsigned short highvsn; /* 0 = OTP-R3 erts-4.6.x, 1 = OTP-R4 erts-4.7.x*/ + unsigned short highvsn; /* 5: creation=1..3, 6: creation=1..(2^32-1)*/ unsigned short lowvsn; int extralen; char extra[MAXSYMLEN+1]; diff --git a/erts/epmd/src/epmd_srv.c b/erts/epmd/src/epmd_srv.c index 3c6f1fbdf4..633ec71e5f 100644 --- a/erts/epmd/src/epmd_srv.c +++ b/erts/epmd/src/epmd_srv.c @@ -665,6 +665,21 @@ static int do_accept(EpmdVars *g,int listensock) return conn_open(g,msgsock); } +static void bump_creation(Node* node) +{ + if (++node->cr_counter == 0) + node->cr_counter = 1; +} +static unsigned int get_creation(Node* node) +{ + if (node->highvsn >= 6) { + return node->cr_counter; /* 1..(2^32-1)*/ + } + else { + return (node->cr_counter - 1) % 3 + 1; /* 1..3 */ + } +} + /* buf is actually one byte larger than bsize, giving place for null termination */ static void do_request(g, fd, s, buf, bsize) @@ -706,8 +721,10 @@ static void do_request(g, fd, s, buf, bsize) unsigned char protocol; unsigned short highvsn; unsigned short lowvsn; + unsigned int creation; int namelen; int extralen; + int replylen; char *name; char *extra; eport = get_int16(&buf[1]); @@ -737,17 +754,22 @@ static void do_request(g, fd, s, buf, bsize) extra = &buf[11+namelen+2]; extra[extralen]='\000'; - wbuf[0] = EPMD_ALIVE2_RESP; - if ((node = node_reg2(g, namelen, name, fd, eport, nodetype, protocol, - highvsn, lowvsn, extralen, extra)) == NULL) { - wbuf[1] = 1; /* error */ - put_int16(99, wbuf+2); - } else { - wbuf[1] = 0; /* ok */ - put_int16(node->creation, wbuf+2); - } + node = node_reg2(g, namelen, name, fd, eport, nodetype, protocol, + highvsn, lowvsn, extralen, extra); + creation = node ? get_creation(node) : 99; + wbuf[1] = node ? 0 : 1; /* ok | error */ + if (highvsn >= 6) { + wbuf[0] = EPMD_ALIVE2_X_RESP; + put_int32(creation, wbuf+2); + replylen = 6; + } + else { + wbuf[0] = EPMD_ALIVE2_RESP; + put_int16(creation, wbuf+2); + replylen = 4; + } - if (!reply(g, fd, wbuf, 4)) + if (!reply(g, fd, wbuf, replylen)) { node_unreg(g, name); dbg_tty_printf(g,1,"** failed to send ALIVE2_RESP for \"%s\"", @@ -1200,8 +1222,8 @@ static int node_unreg(EpmdVars *g,char *name) for (; node; prev = &node->next, node = node->next) if (is_same_str(node->symname, name)) { - dbg_tty_printf(g,1,"unregistering '%s:%d', port %d", - node->symname, node->creation, node->port); + dbg_tty_printf(g,1,"unregistering '%s:%u', port %d", + node->symname, get_creation(node), node->port); *prev = node->next; /* Link out from "reg" list */ @@ -1235,8 +1257,8 @@ static int node_unreg_sock(EpmdVars *g,int fd) for (; node; prev = &node->next, node = node->next) if (node->fd == fd) { - dbg_tty_printf(g,1,"unregistering '%s:%d', port %d", - node->symname, node->creation, node->port); + dbg_tty_printf(g,1,"unregistering '%s:%u', port %d", + node->symname, get_creation(node), node->port); *prev = node->next; /* Link out from "reg" list */ @@ -1264,19 +1286,8 @@ static int node_unreg_sock(EpmdVars *g,int fd) } /* - * Finding a node slot and a (name,creation) name is a bit tricky. - * We try in order - * - * 1. If the name was used before and we can reuse that slot but use - * a new "creation" digit in the range 1..3. - * - * 2. We try to find a new unused slot. - * - * 3. We try to use an used slot this isn't used any longer. - * FIXME: The criteria for *what* slot to steal should be improved. - * Perhaps use the oldest or something. + * Register a new node */ - static Node *node_reg2(EpmdVars *g, int namelen, char* name, @@ -1346,7 +1357,7 @@ static Node *node_reg2(EpmdVars *g, } /* Try to find the name in the used queue so that we - can change "creation" number 1..3 */ + can change "creation" number */ prev = NULL; @@ -1375,9 +1386,8 @@ static Node *node_reg2(EpmdVars *g, g->nodes.unreg_count--; - /* When reusing we change the "creation" number 1..3 */ - - node->creation = node->creation % 3 + 1; + /* When reusing we change the "creation" number */ + bump_creation(node); break; } @@ -1404,7 +1414,8 @@ static Node *node_reg2(EpmdVars *g, exit(1); } - node->creation = (current_time(g) % 3) + 1; /* "random" 1-3 */ + node->cr_counter = current_time(g); /* "random" */ + bump_creation(node); } } @@ -1423,11 +1434,11 @@ static Node *node_reg2(EpmdVars *g, select_fd_set(g, fd); if (highvsn == 0) { - dbg_tty_printf(g,1,"registering '%s:%d', port %d", - node->symname, node->creation, node->port); + dbg_tty_printf(g,1,"registering '%s:%u', port %d", + node->symname, get_creation(node), node->port); } else { - dbg_tty_printf(g,1,"registering '%s:%d', port %d", - node->symname, node->creation, node->port); + dbg_tty_printf(g,1,"registering '%s:%u', port %d", + node->symname, get_creation(node), node->port); dbg_tty_printf(g,1,"type %d proto %d highvsn %d lowvsn %d", nodetype, protocol, highvsn, lowvsn); } @@ -1561,8 +1572,8 @@ static void print_names(EpmdVars *g) for (node = g->nodes.reg; node; node = node->next) { - fprintf(stderr,"***** active name \"%s#%d\" at port %d, fd = %d\r\n", - node->symname, node->creation, node->port, node->fd); + fprintf(stderr,"***** active name \"%s#%u\" at port %d, fd = %d\r\n", + node->symname, get_creation(node), node->port, node->fd); count ++; } @@ -1572,8 +1583,8 @@ static void print_names(EpmdVars *g) for (node = g->nodes.unreg; node; node = node->next) { - fprintf(stderr,"***** old/unused name \"%s#%d\"\r\n", - node->symname, node->creation); + fprintf(stderr,"***** old/unused name \"%s#%u\"\r\n", + node->symname, get_creation(node)); count ++; } diff --git a/erts/etc/unix/etp-commands.in b/erts/etc/unix/etp-commands.in index 20809d61e8..283f859651 100644 --- a/erts/etc/unix/etp-commands.in +++ b/erts/etc/unix/etp-commands.in @@ -1187,12 +1187,14 @@ document etp-cp %--------------------------------------------------------------------------- % etp-cp Eterm % -% Take a code continuation pointer and print +% Take a code continuation pointer or instruction pointer and print % module, function, arity and offset. % -% Code continuation pointers can be found in the process structure e.g -% process_tab[i]->cp and process_tab[i]->i, the second is the -% program counter, which is the same thing as a continuation pointer. +% Code continuation pointers can be found on the stack. The instruction +% pointer is found in the process struct. For example: +% +% c_p->i +% process_tab[i]->i %--------------------------------------------------------------------------- end @@ -1462,10 +1464,6 @@ define etp-stack-preamble printf "I: " etp ((Eterm)($arg0)->i) end - if ($arg0)->cp != 0 - printf "cp:" - etp ((Eterm)($arg0)->cp) - end end define etp-stack-preamble-emu @@ -1474,10 +1472,6 @@ define etp-stack-preamble-emu printf "%% Stacktrace (%u)\n", $etp_stack_end-$etp_stack_p printf "I: " etp ((BeamInstr)I) - if ($arg0)->cp != 0 - printf "cp: " - etp ((Eterm)($arg0)->cp) - end end define etp-stacktrace-1 @@ -2183,13 +2177,6 @@ define etp-process-info-int else printf "unknown\n" end - printf " CP: " - if ($etp_proc->cp) - etp-cp-1 $etp_proc->cp - printf "\n" - else - printf "unknown\n" - end printf " I: " if ($etp_proc->i) etp-cp-1 $etp_proc->i @@ -2419,11 +2406,6 @@ define etp-process-memory-info end end - if ($etp_pmem_proc->cp) - printf " CP: " - etp-cp-1 $etp_pmem_proc->cp - printf " " - end printf "\n" end end diff --git a/erts/etc/unix/run_erl.c b/erts/etc/unix/run_erl.c index bfb3e1bd2c..e974630695 100644 --- a/erts/etc/unix/run_erl.c +++ b/erts/etc/unix/run_erl.c @@ -1201,7 +1201,19 @@ static void error_logf(int priority, int line, const char *format, ...) #ifdef HAVE_SYSLOG_H if (run_daemon) { +#ifdef HAVE_VSYSLOG vsyslog(priority,format,args); +#else + /* Some OSes like AIX lack vsyslog. */ + va_list ap; + char message[900]; /* AIX man page says truncation past this */ + + va_start (ap, format); + vsnprintf(message, 900, format, ap); + va_end(ap); + + syslog(priority, message); +#endif } else #endif diff --git a/erts/lib_src/common/ethr_aux.c b/erts/lib_src/common/ethr_aux.c index 7b156fe01a..931469b386 100644 --- a/erts/lib_src/common/ethr_aux.c +++ b/erts/lib_src/common/ethr_aux.c @@ -109,7 +109,8 @@ x86_init(void) if (eax > 0 && (ETHR_IS_X86_VENDOR("GenuineIntel", ebx, ecx, edx) - || ETHR_IS_X86_VENDOR("AuthenticAMD", ebx, ecx, edx))) { + || ETHR_IS_X86_VENDOR("AuthenticAMD", ebx, ecx, edx) + || ETHR_IS_X86_VENDOR("HygonGenuine", ebx, ecx, edx))) { eax = 1; ethr_x86_cpuid__(&eax, &ebx, &ecx, &edx); } diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam Binary files differindex 866f9df79f..1530982393 100644 --- a/erts/preloaded/ebin/erlang.beam +++ b/erts/preloaded/ebin/erlang.beam diff --git a/erts/preloaded/ebin/prim_file.beam b/erts/preloaded/ebin/prim_file.beam Binary files differindex a2c5f2f336..0152a6091f 100644 --- a/erts/preloaded/ebin/prim_file.beam +++ b/erts/preloaded/ebin/prim_file.beam diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl index 06f0ee1dc6..94595ab78f 100644 --- a/erts/preloaded/src/erlang.erl +++ b/erts/preloaded/src/erlang.erl @@ -109,8 +109,11 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -export([adler32/1, adler32/2, adler32_combine/3, append_element/2]). --export([atom_to_binary/2, atom_to_list/1, binary_part/2, binary_part/3]). --export([binary_to_atom/2, binary_to_existing_atom/2, binary_to_float/1]). +-export([atom_to_binary/1, atom_to_binary/2]). +-export([atom_to_list/1, binary_part/2, binary_part/3]). +-export([binary_to_atom/1, binary_to_atom/2]). +-export([binary_to_existing_atom/1, binary_to_existing_atom/2]). +-export([binary_to_float/1]). -export([binary_to_integer/1,binary_to_integer/2]). -export([binary_to_list/1]). -export([binary_to_list/3, binary_to_term/1, binary_to_term/2]). @@ -186,6 +189,27 @@ -export([dt_get_tag/0, dt_get_tag_data/0, dt_prepend_vm_tag_data/1, dt_append_vm_tag_data/1, dt_put_tag/1, dt_restore_tag/1, dt_spread_tag/1]). +%% Operators + +-export(['=='/2, '=:='/2, + '/='/2, '=/='/2, + '=<'/2, '>='/2, + '<'/2, '>'/2]). + +-export(['-'/1, '+'/1, + '-'/2, '+'/2, + '/'/2, '*'/2, + 'div'/2, 'rem'/2, + 'bsl'/2, 'bsr'/2, + 'bor'/2, 'band'/2, + 'bxor'/2, 'bnot'/1]). + +-export(['and'/2, 'or'/2, + 'xor'/2, 'not'/1]). + +-export(['--'/2, '++'/2]). + +-export(['!'/2]). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%% Simple native code BIFs @@ -341,6 +365,12 @@ adler32_combine(_FirstAdler, _SecondAdler, _SecondSize) -> append_element(_Tuple1, _Term) -> erlang:nif_error(undefined). +%% atom_to_binary/1 +-spec atom_to_binary(Atom) -> binary() when + Atom :: atom(). +atom_to_binary(Atom) -> + erlang:atom_to_binary(Atom, utf8). + %% atom_to_binary/2 -spec atom_to_binary(Atom, Encoding) -> binary() when Atom :: atom(), @@ -371,6 +401,12 @@ binary_part(_Subject, _PosLen) -> binary_part(_Subject, _Start, _Length) -> erlang:nif_error(undefined). +%% binary_to_atom/1 +-spec binary_to_atom(Binary) -> atom() when + Binary :: binary(). +binary_to_atom(Binary) -> + erlang:binary_to_atom(Binary, utf8). + %% binary_to_atom/2 -spec binary_to_atom(Binary, Encoding) -> atom() when Binary :: binary(), @@ -378,6 +414,12 @@ binary_part(_Subject, _Start, _Length) -> binary_to_atom(_Binary, _Encoding) -> erlang:nif_error(undefined). +%% binary_to_existing_atom/1 +-spec binary_to_existing_atom(Binary) -> atom() when + Binary :: binary(). +binary_to_existing_atom(Binary) -> + erlang:binary_to_existing_atom(Binary, utf8). + %% binary_to_existing_atom/2 -spec binary_to_existing_atom(Binary, Encoding) -> atom() when Binary :: binary(), @@ -2186,7 +2228,7 @@ nodes(_Arg) -> -spec open_port(PortName, PortSettings) -> port() when PortName :: {spawn, Command :: string() | binary()} | {spawn_driver, Command :: string() | binary()} | - {spawn_executable, FileName :: file:name() } | + {spawn_executable, FileName :: file:name_all() } | {fd, In :: non_neg_integer(), Out :: non_neg_integer()}, PortSettings :: [Opt], Opt :: {packet, N :: 1 | 2 | 4} @@ -3911,3 +3953,98 @@ gc_info(Ref, N, {OrigColls,OrigRecl}) -> {Ref, {_,Colls, Recl}} -> gc_info(Ref, N-1, {Colls+OrigColls,Recl+OrigRecl}) end. + +%% Operators + +-spec erlang:'=='(term(), term()) -> boolean(). +'=='(_A, _B) -> + erlang:nif_error(undefined). +-spec erlang:'=:='(term(), term()) -> boolean(). +'=:='(_A, _B) -> + erlang:nif_error(undefined). +-spec erlang:'/='(term(), term()) -> boolean(). +'/='(_A, _B) -> + erlang:nif_error(undefined). +-spec erlang:'=/='(term(), term()) -> boolean(). +'=/='(_A, _B) -> + erlang:nif_error(undefined). +-spec erlang:'=<'(term(), term()) -> boolean(). +'=<'(_A, _B) -> + erlang:nif_error(undefined). +-spec erlang:'>='(term(), term()) -> boolean(). +'>='(_A, _B) -> + erlang:nif_error(undefined). +-spec erlang:'<'(term(), term()) -> boolean(). +'<'(_A, _B) -> + erlang:nif_error(undefined). +-spec erlang:'>'(term(), term()) -> boolean(). +'>'(_A, _B) -> + erlang:nif_error(undefined). + +-spec erlang:'-'(number()) -> number(). +'-'(_A) -> + erlang:nif_error(undefined). +-spec erlang:'+'(number()) -> number(). +'+'(_A) -> + erlang:nif_error(undefined). +-spec erlang:'-'(number(), number()) -> number(). +'-'(_A, _B) -> + erlang:nif_error(undefined). +-spec erlang:'+'(number(), number()) -> number(). +'+'(_A, _B) -> + erlang:nif_error(undefined). +-spec erlang:'/'(number(), number()) -> float(). +'/'(_A, _B) -> + erlang:nif_error(undefined). +-spec erlang:'*'(number(), number()) -> number(). +'*'(_A, _B) -> + erlang:nif_error(undefined). +-spec erlang:'div'(integer(), integer()) -> integer(). +'div'(_A, _B) -> + erlang:nif_error(undefined). +-spec erlang:'rem'(integer(), integer()) -> integer(). +'rem'(_A, _B) -> + erlang:nif_error(undefined). +-spec erlang:'bsl'(integer(), integer()) -> integer(). +'bsl'(_A, _B) -> + erlang:nif_error(undefined). +-spec erlang:'bsr'(integer(), integer()) -> integer(). +'bsr'(_A, _B) -> + erlang:nif_error(undefined). +-spec erlang:'bor'(integer(), integer()) -> integer(). +'bor'(_A, _B) -> + erlang:nif_error(undefined). +-spec erlang:'band'(integer(), integer()) -> integer(). +'band'(_A, _B) -> + erlang:nif_error(undefined). +-spec erlang:'bxor'(integer(), integer()) -> integer(). +'bxor'(_A, _B) -> + erlang:nif_error(undefined). +-spec erlang:'bnot'(integer()) -> integer(). +'bnot'(_A) -> + erlang:nif_error(undefined). + +-spec erlang:'--'(list(), list()) -> list(). +'--'(_A, _B) -> + erlang:nif_error(undefined). +-spec erlang:'++'(list(), term()) -> term(). +'++'(_A, _B) -> + erlang:nif_error(undefined). + +-spec erlang:'and'(boolean(), boolean()) -> boolean(). +'and'(_A, _B) -> + erlang:nif_error(undefined). +-spec erlang:'or'(boolean(), boolean()) -> boolean(). +'or'(_A, _B) -> + erlang:nif_error(undefined). + +-spec erlang:'xor'(boolean(), boolean()) -> boolean(). +'xor'(_A, _B) -> + erlang:nif_error(undefined). +-spec erlang:'not'(boolean()) -> boolean(). +'not'(_A) -> + erlang:nif_error(undefined). + +-spec erlang:'!'(dst(), term()) -> term(). +'!'(_Dst, _Msg) -> + erlang:nif_error(undefined). diff --git a/erts/preloaded/src/prim_file.erl b/erts/preloaded/src/prim_file.erl index 1aa5d85c64..1982424191 100644 --- a/erts/preloaded/src/prim_file.erl +++ b/erts/preloaded/src/prim_file.erl @@ -34,6 +34,7 @@ -export([read_link/1, read_link_all/1, read_link_info/1, read_link_info/2, read_file_info/1, read_file_info/2, + read_handle_info/1, read_handle_info/2, write_file_info/2, write_file_info/3]). -export([list_dir/1, list_dir_all/1]). @@ -497,6 +498,8 @@ get_handle_nif(_FileRef) -> erlang:nif_error(undef). delayed_close_nif(_FileRef) -> erlang:nif_error(undef). +read_handle_info_nif(_FileRef) -> + erlang:nif_error(undef). %% %% Quality-of-life helpers @@ -598,20 +601,37 @@ read_link_info(Name, Opts) -> read_info_1(Name, FollowLinks, TimeType) -> try case read_info_nif(encode_path(Name), FollowLinks) of - {error, Reason} -> - {error, Reason}; - FileInfo -> - CTime = from_posix_seconds(FileInfo#file_info.ctime, TimeType), - MTime = from_posix_seconds(FileInfo#file_info.mtime, TimeType), - ATime = from_posix_seconds(FileInfo#file_info.atime, TimeType), - {ok, FileInfo#file_info{ ctime = CTime, - mtime = MTime, - atime = ATime }} + {error, Reason} -> {error, Reason}; + FileInfo -> {ok, adjust_times(FileInfo, TimeType)} + end + catch + error:_ -> {error, badarg} + end. + +read_handle_info(Fd) -> + read_handle_info_1(Fd, local). +read_handle_info(Fd, Opts) -> + read_handle_info_1(Fd, proplist_get_value(time, Opts, local)). + +read_handle_info_1(Fd, TimeType) -> + try + #{ handle := FRef } = get_fd_data(Fd), + case read_handle_info_nif(FRef) of + {error, Reason} -> {error, Reason}; + FileInfo -> {ok, adjust_times(FileInfo, TimeType)} end catch error:_ -> {error, badarg} end. +adjust_times(FileInfo, TimeType) -> + CTime = from_posix_seconds(FileInfo#file_info.ctime, TimeType), + MTime = from_posix_seconds(FileInfo#file_info.mtime, TimeType), + ATime = from_posix_seconds(FileInfo#file_info.atime, TimeType), + FileInfo#file_info{ ctime = CTime, + mtime = MTime, + atime = ATime }. + write_file_info(Filename, Info) -> write_file_info_1(Filename, Info, local). write_file_info(Filename, Info, Opts) -> diff --git a/erts/test/erl_print_SUITE.erl b/erts/test/erl_print_SUITE.erl index 463d890688..0a5987df88 100644 --- a/erts/test/erl_print_SUITE.erl +++ b/erts/test/erl_print_SUITE.erl @@ -324,6 +324,9 @@ run_case(Config, TestArgs, Fun) -> -define(PORT_EXT, 102). -define(PID_EXT, 103). -define(NEW_REFERENCE_EXT, 114). +-define(NEW_PID_EXT, $X). +-define(NEW_PORT_EXT, $Y). +-define(NEWER_REFERENCE_EXT, $Z). uint32_be(Uint) when is_integer(Uint), 0 =< Uint, Uint < 1 bsl 32 -> [(Uint bsr 24) band 16#ff, @@ -351,13 +354,13 @@ mk_pid({NodeName, Creation}, Number, Serial) when is_atom(NodeName) -> mk_pid({atom_to_list(NodeName), Creation}, Number, Serial); mk_pid({NodeName, Creation}, Number, Serial) -> case catch binary_to_term(list_to_binary([?VERSION_MAGIC, - ?PID_EXT, + ?NEW_PID_EXT, ?ATOM_EXT, uint16_be(length(NodeName)), NodeName, uint32_be(Number), uint32_be(Serial), - uint8(Creation)])) of + uint32_be(Creation)])) of Pid when is_pid(Pid) -> Pid; {'EXIT', {badarg, _}} -> @@ -370,12 +373,12 @@ mk_port({NodeName, Creation}, Number) when is_atom(NodeName) -> mk_port({atom_to_list(NodeName), Creation}, Number); mk_port({NodeName, Creation}, Number) -> case catch binary_to_term(list_to_binary([?VERSION_MAGIC, - ?PORT_EXT, + ?NEW_PORT_EXT, ?ATOM_EXT, uint16_be(length(NodeName)), NodeName, uint32_be(Number), - uint8(Creation)])) of + uint32_be(Creation)])) of Port when is_port(Port) -> Port; {'EXIT', {badarg, _}} -> @@ -388,33 +391,16 @@ mk_ref({NodeName, Creation}, Numbers) when is_atom(NodeName), is_integer(Creation), is_list(Numbers) -> mk_ref({atom_to_list(NodeName), Creation}, Numbers); -mk_ref({NodeName, Creation}, [Number]) when is_list(NodeName), - is_integer(Creation), - is_integer(Number) -> - case catch binary_to_term(list_to_binary([?VERSION_MAGIC, - ?REFERENCE_EXT, - ?ATOM_EXT, - uint16_be(length(NodeName)), - NodeName, - uint32_be(Number), - uint8(Creation)])) of - Ref when is_reference(Ref) -> - Ref; - {'EXIT', {badarg, _}} -> - exit({badarg, mk_ref, [{NodeName, Creation}, [Number]]}); - Other -> - exit({unexpected_binary_to_term_result, Other}) - end; mk_ref({NodeName, Creation}, Numbers) when is_list(NodeName), is_integer(Creation), is_list(Numbers) -> case catch binary_to_term(list_to_binary([?VERSION_MAGIC, - ?NEW_REFERENCE_EXT, + ?NEWER_REFERENCE_EXT, uint16_be(length(Numbers)), ?ATOM_EXT, uint16_be(length(NodeName)), NodeName, - uint8(Creation), + uint32_be(Creation), lists:map(fun (N) -> uint32_be(N) end, @@ -429,11 +415,10 @@ mk_ref({NodeName, Creation}, Numbers) when is_list(NodeName), my_cre() -> erlang:system_info(creation). -oth_cre(0) -> 1; -oth_cre(1) -> 2; -oth_cre(2) -> 3; -oth_cre(3) -> 1; -oth_cre(N) -> exit({invalid_creation, N}). +oth_cre(N) when N >= 0, N < (1 bsl 32) -> + (N rem ((1 bsl 32) - 1)) + 1; +oth_cre(N) -> + exit({invalid_creation, N}). str_1_bsl_10000() -> "19950631168807583848837421626835850838234968318861924548520089498529438830221946631919961684036194597899331129423209124271556491349413781117593785932096323957855730046793794526765246551266059895520550086918193311542508608460618104685509074866089624888090489894838009253941633257850621568309473902556912388065225096643874441046759871626985453222868538161694315775629640762836880760732228535091641476183956381458969463899410840960536267821064621427333394036525565649530603142680234969400335934316651459297773279665775606172582031407994198179607378245683762280037302885487251900834464581454650557929601414833921615734588139257095379769119277800826957735674444123062018757836325502728323789270710373802866393031428133241401624195671690574061419654342324638801248856147305207431992259611796250130992860241708340807605932320161268492288496255841312844061536738951487114256315111089745514203313820202931640957596464756010405845841566072044962867016515061920631004186422275908670900574606417856951911456055068251250406007519842261898059237118054444788072906395242548339221982707404473162376760846613033778706039803413197133493654622700563169937455508241780972810983291314403571877524768509857276937926433221599399876886660808368837838027643282775172273657572744784112294389733810861607423253291974813120197604178281965697475898164531258434135959862784130128185406283476649088690521047580882615823961985770122407044330583075869039319604603404973156583208672105913300903752823415539745394397715257455290510212310947321610753474825740775273986348298498340756937955646638621874569499279016572103701364433135817214311791398222983845847334440270964182851005072927748364550578634501100852987812389473928699540834346158807043959118985815145779177143619698728131459483783202081474982171858011389071228250905826817436220577475921417653715687725614904582904992461028630081535583308130101987675856234343538955409175623400844887526162643568648833519463720377293240094456246923254350400678027273837755376406726898636241037491410966718557050759098100246789880178271925953381282421954028302759408448955014676668389697996886241636313376393903373455801407636741877711055384225739499110186468219696581651485130494222369947714763069155468217682876200362777257723781365331611196811280792669481887201298643660768551639860534602297871557517947385246369446923087894265948217008051120322365496288169035739121368338393591756418733850510970271613915439590991598154654417336311656936031122249937969999226781732358023111862644575299135758175008199839236284615249881088960232244362173771618086357015468484058622329792853875623486556440536962622018963571028812361567512543338303270029097668650568557157505516727518899194129711337690149916181315171544007728650573189557450920330185304847113818315407324053319038462084036421763703911550639789000742853672196280903477974533320468368795868580237952218629120080742819551317948157624448298518461509704888027274721574688131594750409732115080498190455803416826949787141316063210686391511681774304792596709376". diff --git a/erts/test/otp_SUITE.erl b/erts/test/otp_SUITE.erl index 2372e8b9ac..c5e0dfe649 100644 --- a/erts/test/otp_SUITE.erl +++ b/erts/test/otp_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2018. All Rights Reserved. +%% Copyright Ericsson AB 2000-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -79,8 +79,7 @@ undefined_functions(Config) when is_list(Config) -> [UndefS,ExcludeFrom]), {ok,Undef0} = xref:q(Server, lists:flatten(Q)), Undef1 = hipe_filter(Undef0), - Undef2 = ssl_crypto_filter(Undef1), - Undef3 = edoc_filter(Undef2), + Undef3 = ssl_crypto_filter(Undef1), Undef4 = eunit_filter(Undef3), Undef5 = dialyzer_filter(Undef4), Undef6 = wx_filter(Undef5), @@ -157,12 +156,6 @@ ssl_crypto_filter(Undef) -> {_,_} -> Undef end. -edoc_filter(Undef) -> - %% Filter away function call that is catched. - filter(fun({{edoc_lib,uri_get_http,1},{http,request_sync,2}}) -> false; - (_) -> true - end, Undef). - eunit_filter(Undef) -> filter(fun({{eunit_test,wrapper_test_exported_,0}, {eunit_test,nonexisting_function,0}}) -> false; diff --git a/lib/asn1/Makefile b/lib/asn1/Makefile index 26e7e37924..10a8795703 100644 --- a/lib/asn1/Makefile +++ b/lib/asn1/Makefile @@ -54,12 +54,7 @@ SPECIAL_TARGETS = include $(ERL_TOP)/make/otp_subdir.mk -.PHONY: info version - -info: - @echo "APP_RELEASE_DIR: $(APP_RELEASE_DIR)" - @echo "APP_DIR: $(APP_DIR)" - @echo "APP_TAR_FILE: $(APP_TAR_FILE)" +.PHONY: version version: @echo "$(VSN)" @@ -100,3 +95,5 @@ tar: $(APP_TAR_FILE) $(APP_TAR_FILE): $(APP_DIR) (cd $(APP_RELEASE_DIR); gtar zcf $(APP_TAR_FILE) $(DIR_NAME)) + +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/common_test/Makefile b/lib/common_test/Makefile index f2065b8a0d..5ac76f0044 100644 --- a/lib/common_test/Makefile +++ b/lib/common_test/Makefile @@ -45,3 +45,7 @@ SPECIAL_TARGETS = # include $(ERL_TOP)/make/otp_subdir.mk +DIA_PLT_APPS=compiler tools crypto runtime_tools syntax_tools ftp inets \ + debugger sasl snmp ssh reltool observer xmerl + +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/common_test/src/ct_property_test.erl b/lib/common_test/src/ct_property_test.erl index f51f454d08..d884a6d494 100644 --- a/lib/common_test/src/ct_property_test.erl +++ b/lib/common_test/src/ct_property_test.erl @@ -32,27 +32,34 @@ %% API -export([init_per_suite/1, - quickcheck/2]). + init_tool/1, + quickcheck/2]). -include_lib("common_test/include/ct.hrl"). init_per_suite(Config) -> + case init_tool(Config) of + {skip, _}=Skip -> + Skip; + Config1 -> + Path = property_tests_path("property_test", Config1), + case compile_tests(Path, Config1) of + error -> + {fail, "Property test compilation failed in "++Path}; + up_to_date -> + add_code_pathz(Path), + [{property_dir, Path} | Config1] + end + end. + +init_tool(Config) -> case which_module_exists([eqc,proper,triq]) of - {ok,ToolModule} -> - ct:pal("Found property tester ~p",[ToolModule]), - Path = property_tests_path("property_test", Config), - case compile_tests(Path,ToolModule) of - error -> - {fail, "Property test compilation failed in "++Path}; - up_to_date -> - add_code_pathz(Path), - [{property_dir,Path}, - {property_test_tool,ToolModule} | Config] - end; - - not_found -> - ct:pal("No property tester found",[]), - {skip, "No property testing tool found"} + {ok, ToolModule} -> + ct:pal("Found property tester ~p",[ToolModule]), + [{property_test_tool, ToolModule} | Config]; + not_found -> + ct:pal("No property tester found",[]), + {skip, "No property testing tool found"} end. quickcheck(Property, Config) -> @@ -105,7 +112,8 @@ add_code_pathz(Dir) -> ok end. -compile_tests(Path, ToolModule) -> +compile_tests(Path, Config) -> + ToolModule = proplists:get_value(property_test_tool, Config), MacroDefs = macro_def(ToolModule), {ok,Cwd} = file:get_cwd(), ok = file:set_cwd(Path), diff --git a/lib/compiler/Makefile b/lib/compiler/Makefile index b8b2f562a2..f18df11e9f 100644 --- a/lib/compiler/Makefile +++ b/lib/compiler/Makefile @@ -36,3 +36,6 @@ SPECIAL_TARGETS = # include $(ERL_TOP)/make/otp_subdir.mk +DIA_PLT_APPS=crypto hipe + +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/compiler/src/Makefile b/lib/compiler/src/Makefile index 87b0d345f2..f253f31d13 100644 --- a/lib/compiler/src/Makefile +++ b/lib/compiler/src/Makefile @@ -49,10 +49,10 @@ MODULES = \ beam_a \ beam_asm \ beam_block \ + beam_call_types \ beam_clean \ beam_dict \ beam_disasm \ - beam_except \ beam_flatten \ beam_jump \ beam_listing \ @@ -72,6 +72,7 @@ MODULES = \ beam_ssa_type \ beam_kernel_to_ssa \ beam_trim \ + beam_types \ beam_utils \ beam_validator \ beam_z \ @@ -104,6 +105,7 @@ HRL_FILES= \ beam_disasm.hrl \ beam_ssa_opt.hrl \ beam_ssa.hrl \ + beam_types.hrl \ core_parse.hrl \ v3_kernel.hrl @@ -190,6 +192,7 @@ release_docs_spec: # Dependencies -- alphabetically, please # ---------------------------------------------------- +$(EBIN)/beam_call_types.beam: beam_types.hrl $(EBIN)/beam_disasm.beam: $(EGEN)/beam_opcodes.hrl beam_disasm.hrl $(EBIN)/beam_listing.beam: core_parse.hrl v3_kernel.hrl beam_ssa.hrl $(EBIN)/beam_kernel_to_ssa.beam: v3_kernel.hrl beam_ssa.hrl @@ -204,7 +207,8 @@ $(EBIN)/beam_ssa_pp.beam: beam_ssa.hrl $(EBIN)/beam_ssa_pre_codegen.beam: beam_ssa.hrl $(EBIN)/beam_ssa_recv.beam: beam_ssa.hrl $(EBIN)/beam_ssa_share.beam: beam_ssa.hrl -$(EBIN)/beam_ssa_type.beam: beam_ssa.hrl +$(EBIN)/beam_ssa_type.beam: beam_ssa.hrl beam_types.hrl +$(EBIN)/beam_types.beam: beam_types.hrl $(EBIN)/cerl.beam: core_parse.hrl $(EBIN)/compile.beam: core_parse.hrl ../../stdlib/include/erl_compile.hrl $(EBIN)/core_lib.beam: core_parse.hrl diff --git a/lib/compiler/src/beam_asm.erl b/lib/compiler/src/beam_asm.erl index df09dcb06c..60e19ec596 100644 --- a/lib/compiler/src/beam_asm.erl +++ b/lib/compiler/src/beam_asm.erl @@ -64,11 +64,30 @@ module(Code, ExtraChunks, CompileInfo, CompilerOpts) -> assemble({Mod,Exp0,Attr0,Asm0,NumLabels}, ExtraChunks, CompileInfo, CompilerOpts) -> {1,Dict0} = beam_dict:atom(Mod, beam_dict:new()), {0,Dict1} = beam_dict:fname(atom_to_list(Mod) ++ ".erl", Dict0), + Dict2 = shared_fun_wrappers(CompilerOpts, Dict1), NumFuncs = length(Asm0), {Asm,Attr} = on_load(Asm0, Attr0), Exp = cerl_sets:from_list(Exp0), - {Code,Dict2} = assemble_1(Asm, Exp, Dict1, []), - build_file(Code, Attr, Dict2, NumLabels, NumFuncs, ExtraChunks, CompileInfo, CompilerOpts). + {Code,Dict} = assemble_1(Asm, Exp, Dict2, []), + build_file(Code, Attr, Dict, NumLabels, NumFuncs, + ExtraChunks, CompileInfo, CompilerOpts). + +shared_fun_wrappers(Opts, Dict) -> + case proplists:get_bool(no_shared_fun_wrappers, Opts) of + false -> + %% The compiler in OTP 23 depends on the on the loader + %% using the new indices in funs and being able to have + %% multiple make_fun2 instructions referring to the same + %% fun entry. Artificially set the highest opcode for the + %% module to ensure that it cannot be loaded in OTP 22 + %% and earlier. + Swap = beam_opcodes:opcode(swap, 2), + beam_dict:opcode(Swap, Dict); + true -> + %% Fun wrappers are not shared for compatibility with a + %% previous OTP release. + Dict + end. on_load(Fs0, Attr0) -> case proplists:get_value(on_load, Attr0) of diff --git a/lib/compiler/src/beam_block.erl b/lib/compiler/src/beam_block.erl index 707974b2c1..a734ca3a10 100644 --- a/lib/compiler/src/beam_block.erl +++ b/lib/compiler/src/beam_block.erl @@ -33,8 +33,9 @@ module({Mod,Exp,Attr,Fs0,Lc}, _Opts) -> function({function,Name,Arity,CLabel,Is0}) -> try - Is1 = blockify(Is0), - Is = embed_lines(Is1), + Is1 = swap_opt(Is0), + Is2 = blockify(Is1), + Is = embed_lines(Is2), {function,Name,Arity,CLabel,Is} catch Class:Error:Stack -> @@ -42,6 +43,40 @@ function({function,Name,Arity,CLabel,Is0}) -> erlang:raise(Class, Error, Stack) end. +%%% +%%% Try to use a `swap` instruction instead of a sequence of moves. +%%% +%%% Note that beam_ssa_codegen generates `swap` instructions only for +%%% the moves within a single SSA instruction (such as `call`), not +%%% for the moves generated by a sequence of SSA instructions. +%%% Therefore, this optimization is needed. +%%% + +swap_opt([{move,Reg1,{x,X}=Temp}=Move1, + {move,Reg2,Reg1}=Move2, + {move,Temp,Reg2}=Move3|Is]) when Reg1 =/= Temp -> + case is_unused(X, Is) of + true -> + [{swap,Reg1,Reg2}|swap_opt(Is)]; + false -> + [Move1|swap_opt([Move2,Move3|Is])] + end; +swap_opt([I|Is]) -> + [I|swap_opt(Is)]; +swap_opt([]) -> []. + +is_unused(X, [{call,A,_}|_]) when A =< X -> true; +is_unused(X, [{call_ext,A,_}|_]) when A =< X -> true; +is_unused(X, [{make_fun2,_,_,_,A}|_]) when A =< X -> true; +is_unused(X, [{move,Src,Dst}|Is]) -> + case {Src,Dst} of + {{x,X},_} -> false; + {_,{x,X}} -> true; + {_,_} -> is_unused(X, Is) + end; +is_unused(X, [{line,_}|Is]) -> is_unused(X, Is); +is_unused(_, _) -> false. + %% blockify(Instructions0) -> Instructions %% Collect sequences of instructions to basic blocks. %% Also do some simple optimations on instructions outside the blocks. diff --git a/lib/compiler/src/beam_call_types.erl b/lib/compiler/src/beam_call_types.erl new file mode 100644 index 0000000000..904d82a62d --- /dev/null +++ b/lib/compiler/src/beam_call_types.erl @@ -0,0 +1,551 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(beam_call_types). + +-include("beam_types.hrl"). + +-import(lists, [duplicate/2,foldl/3]). + +-export([will_succeed/3, types/3]). + +%% +%% Returns whether a call will succeed or not. +%% +%% Note that it only answers 'yes' for functions in the 'erlang' module as +%% calls to other modules may fail due to not being loaded, even if we consider +%% the module to be known. +%% + +-spec will_succeed(Mod, Func, ArgTypes) -> Result when + Mod :: atom(), + Func :: atom(), + ArgTypes :: [normal_type()], + Result :: yes | no | maybe. + +will_succeed(erlang, BoolOp, [LHS, RHS]) when BoolOp =:= 'and'; + BoolOp =:= 'or' -> + case {succeeds_if_type(LHS, beam_types:make_boolean()), + succeeds_if_type(RHS, beam_types:make_boolean())} of + {yes, yes} -> yes; + {no, _} -> no; + {_, no} -> no; + {_, _} -> maybe + end; +will_succeed(erlang, bit_size, [Arg]) -> + succeeds_if_type(Arg, #t_bitstring{}); +will_succeed(erlang, byte_size, [Arg]) -> + succeeds_if_type(Arg, #t_bitstring{}); +will_succeed(erlang, map_size, [Arg]) -> + succeeds_if_type(Arg, #t_map{}); +will_succeed(erlang, 'not', [Arg]) -> + succeeds_if_type(Arg, beam_types:make_boolean()); +will_succeed(erlang, setelement, [#t_integer{elements={Min,Max}}, + #t_tuple{exact=Exact,size=Size}, _]) -> + case Min >= 1 andalso Max =< Size of + true -> yes; + false when Exact -> no; + false -> maybe + end; +will_succeed(erlang, size, [Arg]) -> + succeeds_if_type(Arg, #t_bitstring{}); +will_succeed(erlang, tuple_size, [Arg]) -> + succeeds_if_type(Arg, #t_tuple{}); +will_succeed(Mod, Func, Args) -> + Arity = length(Args), + case erl_bifs:is_safe(Mod, Func, Arity) of + true -> + yes; + false -> + case erl_bifs:is_exit_bif(Mod, Func, Arity) of + true -> no; + false -> maybe + end + end. + +succeeds_if_type(ArgType, Required) -> + case beam_types:meet(ArgType, Required) of + ArgType -> yes; + none -> no; + _ -> maybe + end. + +%% +%% Returns the inferred return and argument types for known functions, and +%% whether it's safe to subtract argument types on failure. +%% +%% Note that the return type will be 'none' if we can statically determine that +%% the function will fail at runtime. +%% + +-spec types(Mod, Func, ArgTypes) -> {RetType, ArgTypes, CanSubtract} when + Mod :: atom(), + Func :: atom(), + ArgTypes :: [normal_type()], + RetType :: type(), + CanSubtract :: boolean(). + +%% Functions that only fail due to bad argument *types*, meaning it's safe to +%% subtract argument types on failure. +%% +%% Note that these are all from the erlang module; suitable functions in other +%% modules could fail due to the module not being loaded. +types(erlang, 'map_size', [_]) -> + sub_safe(#t_integer{}, [#t_map{}]); +types(erlang, 'tuple_size', [_]) -> + sub_safe(#t_integer{}, [#t_tuple{}]); +types(erlang, 'bit_size', [_]) -> + sub_safe(#t_integer{}, [#t_bitstring{}]); +types(erlang, 'byte_size', [_]) -> + sub_safe(#t_integer{}, [#t_bitstring{}]); +types(erlang, 'hd', [_]) -> + sub_safe(any, [cons]); +types(erlang, 'tl', [_]) -> + sub_safe(any, [cons]); +types(erlang, 'length', [_]) -> + sub_safe(#t_integer{}, [list]); +types(erlang, 'not', [_]) -> + Bool = beam_types:make_boolean(), + sub_safe(Bool, [Bool]); + +%% Boolean ops +types(erlang, 'and', [_,_]) -> + Bool = beam_types:make_boolean(), + sub_unsafe(Bool, [Bool, Bool]); +types(erlang, 'or', [_,_]) -> + Bool = beam_types:make_boolean(), + sub_unsafe(Bool, [Bool, Bool]); +types(erlang, 'xor', [_,_]) -> + Bool = beam_types:make_boolean(), + sub_unsafe(Bool, [Bool, Bool]); + +%% Bitwise ops +types(erlang, 'band', [_,_]=Args) -> + sub_unsafe(band_return_type(Args), [#t_integer{}, #t_integer{}]); +types(erlang, 'bor', [_,_]) -> + sub_unsafe(#t_integer{}, [#t_integer{}, #t_integer{}]); +types(erlang, 'bxor', [_,_]) -> + sub_unsafe(#t_integer{}, [#t_integer{}, #t_integer{}]); +types(erlang, 'bsl', [_,_]) -> + sub_unsafe(#t_integer{}, [#t_integer{}, #t_integer{}]); +types(erlang, 'bsr', [_,_]) -> + sub_unsafe(#t_integer{}, [#t_integer{}, #t_integer{}]); +types(erlang, 'bnot', [_]) -> + sub_unsafe(#t_integer{}, [#t_integer{}]); + +%% Fixed-type arithmetic +types(erlang, 'float', [_]) -> + sub_unsafe(float, [number]); +types(erlang, 'round', [_]) -> + sub_unsafe(#t_integer{}, [number]); +types(erlang, 'floor', [_]) -> + sub_unsafe(#t_integer{}, [number]); +types(erlang, 'ceil', [_]) -> + sub_unsafe(#t_integer{}, [number]); +types(erlang, 'trunc', [_]) -> + sub_unsafe(#t_integer{}, [number]); +types(erlang, '/', [_,_]) -> + sub_unsafe(float, [number, number]); +types(erlang, 'div', [_,_]) -> + sub_unsafe(#t_integer{}, [#t_integer{}, #t_integer{}]); +types(erlang, 'rem', [_,_]) -> + sub_unsafe(#t_integer{}, [#t_integer{}, #t_integer{}]); + +%% Mixed-type arithmetic; '+'/2 and friends are handled in the catch-all +%% clause for the 'erlang' module. +types(erlang, 'abs', [_]=Args) -> + mixed_arith_types(Args); + +%% List operations +types(erlang, '++', [LHS,RHS]) -> + %% `[] ++ RHS` yields RHS, even if RHS is not a list. + RetType = case {LHS, RHS} of + {cons, _} -> cons; + {_, cons} -> cons; + _ -> beam_types:join(list, RHS) + end, + sub_unsafe(RetType, [list, any]); +types(erlang, '--', [_,_]) -> + sub_unsafe(list, [list, list]); + +%% Misc ops. +types(erlang, 'binary_part', [_, _]) -> + PosLen = make_two_tuple(#t_integer{}, #t_integer{}), + Binary = #t_bitstring{unit=8}, + sub_unsafe(Binary, [Binary, PosLen]); +types(erlang, 'binary_part', [_, _, _]) -> + Binary = #t_bitstring{unit=8}, + sub_unsafe(Binary, [Binary, #t_integer{}, #t_integer{}]); +types(erlang, 'is_map_key', [_,_]) -> + sub_unsafe(beam_types:make_boolean(), [any,#t_map{}]); +types(erlang, 'map_get', [_,_]) -> + sub_unsafe(any, [any,#t_map{}]); +types(erlang, 'node', [_]) -> + sub_unsafe(#t_atom{}, [any]); +types(erlang, 'node', []) -> + sub_unsafe(#t_atom{}, []); +types(erlang, 'size', [_]) -> + sub_unsafe(#t_integer{}, [any]); +types(erlang, 'size', [_]) -> + sub_unsafe(#t_integer{}, [any]); + +%% Tuple element ops +types(erlang, element, [PosType, TupleType]) -> + Index = case PosType of + #t_integer{elements={Same,Same}} when is_integer(Same) -> + Same; + _ -> + 0 + end, + + RetType = case TupleType of + #t_tuple{size=Sz,elements=Es} when Index =< Sz, + Index >= 1 -> + beam_types:get_element_type(Index, Es); + _ -> + any + end, + + sub_unsafe(RetType, [#t_integer{}, #t_tuple{size=Index}]); +types(erlang, setelement, [PosType, TupleType, ArgType]) -> + RetType = case {PosType,TupleType} of + {#t_integer{elements={Index,Index}}, + #t_tuple{elements=Es0,size=Size}=T} when Index >= 1 -> + %% This is an exact index, update the type of said + %% element or return 'none' if it's known to be out of + %% bounds. + Es = beam_types:set_element_type(Index, ArgType, Es0), + case T#t_tuple.exact of + false -> + T#t_tuple{size=max(Index, Size),elements=Es}; + true when Index =< Size -> + T#t_tuple{elements=Es}; + true -> + none + end; + {#t_integer{elements={Min,Max}}, + #t_tuple{elements=Es0,size=Size}=T} when Min >= 1 -> + %% We know this will land between Min and Max, so kill + %% the types for those indexes. + Es = discard_tuple_element_info(Min, Max, Es0), + case T#t_tuple.exact of + false -> + T#t_tuple{elements=Es,size=max(Min, Size)}; + true when Min =< Size -> + T#t_tuple{elements=Es,size=Size}; + true -> + none + end; + {_,#t_tuple{}=T} -> + %% Position unknown, so we have to discard all element + %% information. + T#t_tuple{elements=#{}}; + {#t_integer{elements={Min,_Max}},_} -> + #t_tuple{size=Min}; + {_,_} -> + #t_tuple{} + end, + sub_unsafe(RetType, [#t_integer{}, #t_tuple{}, any]); + +types(erlang, make_fun, [_,_,Arity0]) -> + Type = case Arity0 of + #t_integer{elements={Arity,Arity}} when Arity >= 0 -> + #t_fun{arity=Arity}; + _ -> + #t_fun{} + end, + sub_unsafe(Type, [#t_atom{}, #t_atom{}, #t_integer{}]); + +types(erlang, Name, Args) -> + Arity = length(Args), + + case erl_bifs:is_exit_bif(erlang, Name, Arity) of + true -> + {none, Args, false}; + false -> + case erl_internal:arith_op(Name, Arity) of + true -> + mixed_arith_types(Args); + false -> + IsTest = + erl_internal:new_type_test(Name, Arity) orelse + erl_internal:comp_op(Name, Arity), + + RetType = case IsTest of + true -> beam_types:make_boolean(); + false -> any + end, + + sub_unsafe(RetType, duplicate(Arity, any)) + end + end; + +%% +%% Math BIFs +%% + +types(math, cos, [_]) -> + sub_unsafe(float, [number]); +types(math, cosh, [_]) -> + sub_unsafe(float, [number]); +types(math, sin, [_]) -> + sub_unsafe(float, [number]); +types(math, sinh, [_]) -> + sub_unsafe(float, [number]); +types(math, tan, [_]) -> + sub_unsafe(float, [number]); +types(math, tanh, [_]) -> + sub_unsafe(float, [number]); +types(math, acos, [_]) -> + sub_unsafe(float, [number]); +types(math, acosh, [_]) -> + sub_unsafe(float, [number]); +types(math, asin, [_]) -> + sub_unsafe(float, [number]); +types(math, asinh, [_]) -> + sub_unsafe(float, [number]); +types(math, atan, [_]) -> + sub_unsafe(float, [number]); +types(math, atanh, [_]) -> + sub_unsafe(float, [number]); +types(math, erf, [_]) -> + sub_unsafe(float, [number]); +types(math, erfc, [_]) -> + sub_unsafe(float, [number]); +types(math, exp, [_]) -> + sub_unsafe(float, [number]); +types(math, log, [_]) -> + sub_unsafe(float, [number]); +types(math, log2, [_]) -> + sub_unsafe(float, [number]); +types(math, log10, [_]) -> + sub_unsafe(float, [number]); +types(math, sqrt, [_]) -> + sub_unsafe(float, [number]); +types(math, atan2, [_,_]) -> + sub_unsafe(float, [number, number]); +types(math, pow, [_,_]) -> + sub_unsafe(float, [number, number]); +types(math, ceil, [_]) -> + sub_unsafe(float, [number]); +types(math, floor, [_]) -> + sub_unsafe(float, [number]); +types(math, fmod, [_,_]) -> + sub_unsafe(float, [number, number]); +types(math, pi, []) -> + sub_unsafe(float, []); + +%% +%% List functions +%% + +%% Operator aliases. +types(lists, append, [_,_]=Args) -> + types(erlang, '++', Args); +types(lists, append, [_]) -> + %% This is implemented through folding the list over erlang:'++'/2, so it + %% can hypothetically return anything, but we can infer that its argument + %% is a list on success. + sub_unsafe(any, [list]); +types(lists, subtract, [_,_]) -> + sub_unsafe(list, [list, list]); + +%% Functions returning booleans. +types(lists, all, [_,_]) -> + sub_unsafe(beam_types:make_boolean(), [#t_fun{arity=1}, list]); +types(lists, any, [_,_]) -> + sub_unsafe(beam_types:make_boolean(), [#t_fun{arity=1}, list]); +types(lists, keymember, [_,_,_]) -> + sub_unsafe(beam_types:make_boolean(), [any, #t_integer{}, list]); +types(lists, member, [_,_]) -> + sub_unsafe(beam_types:make_boolean(), [any, list]); +types(lists, prefix, [_,_]) -> + sub_unsafe(beam_types:make_boolean(), [list, list]); +types(lists, suffix, [_,_]) -> + sub_unsafe(beam_types:make_boolean(), [list, list]); + +%% Functions returning plain lists. +types(lists, dropwhile, [_,_]) -> + sub_unsafe(list, [#t_fun{arity=1}, list]); +types(lists, duplicate, [_,_]) -> + sub_unsafe(list, [#t_integer{}, any]); +types(lists, filter, [_,_]) -> + sub_unsafe(list, [#t_fun{arity=1}, list]); +types(lists, flatten, [_]) -> + sub_unsafe(list, [list]); +types(lists, map, [_Fun, List]) -> + sub_unsafe(same_length_type(List), [#t_fun{arity=1}, list]); +types(lists, reverse, [List]) -> + sub_unsafe(same_length_type(List), [list]); +types(lists, sort, [List]) -> + sub_unsafe(same_length_type(List), [list]); +types(lists, takewhile, [_,_]) -> + sub_unsafe(list, [#t_fun{arity=1}, list]); +types(lists, usort, [List]) -> + sub_unsafe(same_length_type(List), [list]); +types(lists, zip, [A,B]) -> + ZipType = lists_zip_type([A,B]), + sub_unsafe(ZipType, [ZipType, ZipType]); +types(lists, zip3, [A,B,C]) -> + ZipType = lists_zip_type([A,B,C]), + sub_unsafe(ZipType, [ZipType, ZipType, ZipType]); +types(lists, zipwith, [_,A,B]) -> + ZipType = lists_zip_type([A,B]), + sub_unsafe(ZipType, [#t_fun{arity=2}, ZipType, ZipType]); +types(lists, zipwith3, [_,A,B,C]) -> + ZipType = lists_zip_type([A,B,C]), + sub_unsafe(ZipType, [#t_fun{arity=3}, ZipType, ZipType, ZipType]); + +%% Functions with complex return values. +types(lists, keyfind, [KeyType,PosType,_]) -> + TupleType = case PosType of + #t_integer{elements={Index,Index}} when is_integer(Index), + Index >= 1 -> + Es = beam_types:set_element_type(Index, KeyType, #{}), + #t_tuple{size=Index,elements=Es}; + _ -> + #t_tuple{} + end, + RetType = beam_types:join(TupleType, beam_types:make_atom(false)), + sub_unsafe(RetType, [any, #t_integer{}, list]); +types(lists, MapFold, [_Fun, _Init, List]) + when MapFold =:= mapfoldl; MapFold =:= mapfoldr -> + RetType = make_two_tuple(same_length_type(List), any), + sub_unsafe(RetType, [#t_fun{arity=2}, any, list]); +types(lists, partition, [_,_]) -> + sub_unsafe(make_two_tuple(list, list), [#t_fun{arity=1}, list]); +types(lists, search, [_,_]) -> + TupleType = make_two_tuple(beam_types:make_atom(value), any), + RetType = beam_types:join(TupleType, beam_types:make_atom(false)), + sub_unsafe(RetType, [#t_fun{arity=1}, list]); +types(lists, splitwith, [_,_]) -> + sub_unsafe(make_two_tuple(list, list), [#t_fun{arity=1}, list]); +types(lists, unzip, [List]) -> + ListType = same_length_type(List), + RetType = make_two_tuple(ListType, ListType), + sub_unsafe(RetType, [list]); + +%% Catch-all clause for unknown functions. + +types(_, _, Args) -> + sub_unsafe(any, [any || _ <- Args]). + +%% +%% Helpers +%% + +sub_unsafe(none, ArgTypes) -> + %% This is known to fail at runtime, but the type optimization pass + %% doesn't yet support cutting a block short at any point, so we + %% pretend it's raining instead. + %% + %% Actual exit BIFs get special treatment in the catch-all clause + %% for the 'erlang' module. + sub_unsafe(any, ArgTypes); +sub_unsafe(RetType, ArgTypes) -> + {RetType, ArgTypes, false}. + +sub_safe(RetType, ArgTypes) -> + {RetType, ArgTypes, true}. + +mixed_arith_types([FirstType | _]=Args0) -> + RetType = foldl(fun(#t_integer{}, #t_integer{}) -> #t_integer{}; + (#t_integer{}, number) -> number; + (#t_integer{}, float) -> float; + (float, #t_integer{}) -> float; + (float, number) -> float; + (float, float) -> float; + (number, #t_integer{}) -> number; + (number, float) -> float; + (number, number) -> number; + (any, _) -> number; + (_, _) -> none + end, FirstType, Args0), + sub_unsafe(RetType, [number || _ <- Args0]). + +band_return_type([#t_integer{elements={Int,Int}}, RHS]) when is_integer(Int) -> + band_return_type_1(RHS, Int); +band_return_type([LHS, #t_integer{elements={Int,Int}}]) when is_integer(Int) -> + band_return_type_1(LHS, Int); +band_return_type(_) -> + #t_integer{}. + +band_return_type_1(LHS, Int) -> + case LHS of + #t_integer{elements={Min0,Max0}} when Max0 - Min0 < 1 bsl 256 -> + {Intersection, Union} = range_masks(Min0, Max0), + + Min = Intersection band Int, + Max = min(Max0, Union band Int), + + #t_integer{elements={Min,Max}}; + _ when Int >= 0 -> + %% The range is either unknown or too wide, conservatively assume + %% that the new range is 0 .. Int. + #t_integer{elements={0,Int}}; + _ when Int < 0 -> + %% We can't infer boundaries when the range is unknown and the + %% other operand is a negative number, as the latter sign-extends + %% to infinity and we can't express an inverted range at the + %% moment (cf. X band -8; either less than -7 or greater than 7). + #t_integer{} + end. + +%% Returns two bitmasks describing all possible values between From and To. +%% +%% The first contains the bits that are common to all values, and the second +%% contains the bits that are set by any value in the range. +range_masks(From, To) when From =< To -> + range_masks_1(From, To, 0, -1, 0). + +range_masks_1(From, To, BitPos, Intersection, Union) when From < To -> + range_masks_1(From + (1 bsl BitPos), To, BitPos + 1, + Intersection band From, Union bor From); +range_masks_1(_From, To, _BitPos, Intersection0, Union0) -> + Intersection = To band Intersection0, + Union = To bor Union0, + {Intersection, Union}. + +discard_tuple_element_info(Min, Max, Es) -> + foldl(fun(El, Acc) when Min =< El, El =< Max -> + maps:remove(El, Acc); + (_El, Acc) -> Acc + end, Es, maps:keys(Es)). + +%% For a lists function that return a list of the same length as the input +%% list, return the type of the list. +same_length_type(cons) -> cons; +same_length_type(nil) -> nil; +same_length_type(_) -> list. + +%% lists:zip/2 and friends only succeed when all arguments have the same +%% length, so if one of them is cons, we can infer that all of them are cons +%% on success. +lists_zip_type(Types) -> + foldl(fun(cons, _) -> cons; + (_, cons) -> cons; + (nil, _) -> nil; + (_, T) -> T + end, list, Types). + +make_two_tuple(Type1, Type2) -> + Es0 = beam_types:set_element_type(1, Type1, #{}), + Es = beam_types:set_element_type(2, Type2, Es0), + #t_tuple{size=2,exact=true,elements=Es}. diff --git a/lib/compiler/src/beam_clean.erl b/lib/compiler/src/beam_clean.erl index 7299654476..6b2b2ce085 100644 --- a/lib/compiler/src/beam_clean.erl +++ b/lib/compiler/src/beam_clean.erl @@ -34,7 +34,8 @@ module({Mod,Exp,Attr,Fs0,_}, Opts) -> Used = find_all_used(WorkList, All, cerl_sets:from_list(WorkList)), Fs1 = remove_unused(Order, Used, All), {Fs2,Lc} = clean_labels(Fs1), - Fs = maybe_remove_lines(Fs2, Opts), + Fs3 = fix_swap(Fs2, Opts), + Fs = maybe_remove_lines(Fs3, Opts), {ok,{Mod,Exp,Attr,Fs,Lc}}. %% Determine the rootset, i.e. exported functions and @@ -137,31 +138,54 @@ function_replace([{function,Name,Arity,Entry,Asm0}|Fs], Dict, Acc) -> function_replace([], _, Acc) -> Acc. %%% +%%% If compatibility with a previous release (OTP 22 or earlier) has +%%% been requested, replace swap instructions with a sequence of moves. +%%% + +fix_swap(Fs, Opts) -> + case proplists:get_bool(no_swap, Opts) of + false -> Fs; + true -> fold_functions(fun swap_moves/1, Fs) + end. + +swap_moves([{swap,Reg1,Reg2}|Is]) -> + Temp = {x,1022}, + [{move,Reg1,Temp},{move,Reg2,Reg1},{move,Temp,Reg2}|swap_moves(Is)]; +swap_moves([I|Is]) -> + [I|swap_moves(Is)]; +swap_moves([]) -> []. + +%%% %%% Remove line instructions if requested. %%% maybe_remove_lines(Fs, Opts) -> case proplists:get_bool(no_line_info, Opts) of false -> Fs; - true -> remove_lines(Fs) + true -> fold_functions(fun remove_lines/1, Fs) end. -remove_lines([{function,N,A,Lbl,Is0}|T]) -> - Is = remove_lines_fun(Is0), - [{function,N,A,Lbl,Is}|remove_lines(T)]; -remove_lines([]) -> []. - -remove_lines_fun([{line,_}|Is]) -> - remove_lines_fun(Is); -remove_lines_fun([{block,Bl0}|Is]) -> +remove_lines([{line,_}|Is]) -> + remove_lines(Is); +remove_lines([{block,Bl0}|Is]) -> Bl = remove_lines_block(Bl0), - [{block,Bl}|remove_lines_fun(Is)]; -remove_lines_fun([I|Is]) -> - [I|remove_lines_fun(Is)]; -remove_lines_fun([]) -> []. + [{block,Bl}|remove_lines(Is)]; +remove_lines([I|Is]) -> + [I|remove_lines(Is)]; +remove_lines([]) -> []. remove_lines_block([{set,_,_,{line,_}}|Is]) -> remove_lines_block(Is); remove_lines_block([I|Is]) -> [I|remove_lines_block(Is)]; remove_lines_block([]) -> []. + + +%%% +%%% Helpers. +%%% + +fold_functions(F, [{function,N,A,Lbl,Is0}|T]) -> + Is = F(Is0), + [{function,N,A,Lbl,Is}|fold_functions(F, T)]; +fold_functions(_F, []) -> []. diff --git a/lib/compiler/src/beam_dict.erl b/lib/compiler/src/beam_dict.erl index b2056332e6..4d0cec857d 100644 --- a/lib/compiler/src/beam_dict.erl +++ b/lib/compiler/src/beam_dict.erl @@ -40,6 +40,7 @@ -type lambda_info() :: {label(),{index(),label(),non_neg_integer()}}. -type lambda_tab() :: {non_neg_integer(),[lambda_info()]}. +-type wrapper() :: #{label() => index()}. -record(asm, {atoms = #{} :: atom_tab(), @@ -48,6 +49,7 @@ imports = gb_trees:empty() :: import_tab(), strings = <<>> :: binary(), %String pool lambdas = {0,[]} :: lambda_tab(), + wrappers = #{} :: wrapper(), literals = dict:new() :: literal_tab(), fnames = #{} :: fname_tab(), lines = #{} :: line_tab(), @@ -147,11 +149,21 @@ string(BinString, Dict) when is_binary(BinString) -> -spec lambda(label(), non_neg_integer(), bdict()) -> {non_neg_integer(), bdict()}. -lambda(Lbl, NumFree, #asm{lambdas={OldIndex,Lambdas0}}=Dict) -> - %% Set Index the same as OldIndex. - Index = OldIndex, - Lambdas = [{Lbl,{Index,Lbl,NumFree}}|Lambdas0], - {OldIndex,Dict#asm{lambdas={OldIndex+1,Lambdas}}}. +lambda(Lbl, NumFree, #asm{wrappers=Wrappers0, + lambdas={OldIndex,Lambdas0}}=Dict) -> + case Wrappers0 of + #{Lbl:=Index} -> + %% OTP 23: There old is a fun entry for this wrapper function. + %% Share the fun entry. + {Index,Dict}; + #{} -> + %% Set Index the same as OldIndex. + Index = OldIndex, + Wrappers = Wrappers0#{Lbl=>Index}, + Lambdas = [{Lbl,{Index,Lbl,NumFree}}|Lambdas0], + {OldIndex,Dict#asm{wrappers=Wrappers, + lambdas={OldIndex+1,Lambdas}}} + end. %% Returns the index for a literal (adding it to the literal table if necessary). %% literal(Literal, Dict) -> {Index,Dict'} diff --git a/lib/compiler/src/beam_disasm.erl b/lib/compiler/src/beam_disasm.erl index 7d048716e4..45b69d7e95 100644 --- a/lib/compiler/src/beam_disasm.erl +++ b/lib/compiler/src/beam_disasm.erl @@ -1123,6 +1123,13 @@ resolve_inst({put_tuple2,[Dst,{{z,1},{u,_},List0}]},_,_,_) -> {put_tuple2,Dst,{list,List}}; %% +%% OTP 23. +%% +resolve_inst({swap,[_,_]=List},_,_,_) -> + [R1,R2] = resolve_args(List), + {swap,R1,R2}; + +%% %% Catches instructions that are not yet handled. %% resolve_inst(X,_,_,_) -> ?exit({resolve_inst,X}). diff --git a/lib/compiler/src/beam_except.erl b/lib/compiler/src/beam_except.erl deleted file mode 100644 index 2b9c1b0cf5..0000000000 --- a/lib/compiler/src/beam_except.erl +++ /dev/null @@ -1,255 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2011-2018. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% - --module(beam_except). --export([module/2]). - -%%% Rewrite certain calls to erlang:error/{1,2} to specialized -%%% instructions: -%%% -%%% erlang:error({badmatch,Value}) => badmatch Value -%%% erlang:error({case_clause,Value}) => case_end Value -%%% erlang:error({try_clause,Value}) => try_case_end Value -%%% erlang:error(if_clause) => if_end -%%% erlang:error(function_clause, Args) => jump FuncInfoLabel -%%% - --import(lists, [reverse/1,reverse/2,seq/2,splitwith/2]). - --spec module(beam_utils:module_code(), [compile:option()]) -> - {'ok',beam_utils:module_code()}. - -module({Mod,Exp,Attr,Fs0,Lc}, _Opt) -> - Fs = [function(F) || F <- Fs0], - {ok,{Mod,Exp,Attr,Fs,Lc}}. - -function({function,Name,Arity,CLabel,Is0}) -> - try - Is = function_1(Is0), - {function,Name,Arity,CLabel,Is} - catch - Class:Error:Stack -> - io:fwrite("Function: ~w/~w\n", [Name,Arity]), - erlang:raise(Class, Error, Stack) - end. - --record(st, - {lbl :: beam_asm:label(), %func_info label - loc :: [_], %location for func_info - arity :: arity() %arity for function - }). - -function_1(Is0) -> - case Is0 of - [{label,Lbl},{line,Loc},{func_info,_,_,Arity}|_] -> - St = #st{lbl=Lbl,loc=Loc,arity=Arity}, - translate(Is0, St, []); - [{label,_}|_] -> - %% No line numbers. The source must be a .S file. - %% There is no need to do anything. - Is0 - end. - -translate([{call_ext,Ar,{extfunc,erlang,error,Ar}}=I|Is], St, Acc) -> - translate_1(Ar, I, Is, St, Acc); -translate([I|Is], St, Acc) -> - translate(Is, St, [I|Acc]); -translate([], _, Acc) -> - reverse(Acc). - -translate_1(Ar, I, Is, #st{arity=Arity}=St, [{line,_}=Line|Acc1]=Acc0) -> - case dig_out(Ar, Arity, Acc1) of - no -> - translate(Is, St, [I|Acc0]); - {yes,function_clause,Acc2} -> - case {Is,Line,St} of - {[return|_],{line,Loc},#st{lbl=Fi,loc=Loc}} -> - Instr = {jump,{f,Fi}}, - translate(Is, St, [Instr|Acc2]); - {_,_,_} -> - %% Not a call_only instruction, or not the same - %% location information as in in the line instruction - %% before the func_info instruction. Not safe - %% to translate to a jump. - translate(Is, St, [I|Acc0]) - end; - {yes,Instr,Acc2} -> - translate(Is, St, [Instr,Line|Acc2]) - end. - -dig_out(1, _Arity, Is) -> - dig_out(Is); -dig_out(2, Arity, Is) -> - dig_out_fc(Arity, Is); -dig_out(_, _, _) -> no. - -dig_out([{block,Bl0}|Is]) -> - case dig_out_block(reverse(Bl0)) of - no -> no; - {yes,What,[]} -> - {yes,What,Is}; - {yes,What,Bl} -> - {yes,What,[{block,Bl}|Is]} - end; -dig_out(_) -> no. - -dig_out_block([{set,[{x,0}],[{atom,if_clause}],move}]) -> - {yes,if_end,[]}; -dig_out_block([{set,[{x,0}],[{literal,{Exc,Value}}],move}|Is]) -> - translate_exception(Exc, {literal,Value}, Is, 0); -dig_out_block([{set,[{x,0}],[{atom,Exc},Value],put_tuple2}|Is]) -> - translate_exception(Exc, Value, Is, 3); -dig_out_block(_) -> no. - -translate_exception(badmatch, Val, Is, Words) -> - {yes,{badmatch,Val},fix_block(Is, Words)}; -translate_exception(case_clause, Val, Is, Words) -> - {yes,{case_end,Val},fix_block(Is, Words)}; -translate_exception(try_clause, Val, Is, Words) -> - {yes,{try_case_end,Val},fix_block(Is, Words)}; -translate_exception(_, _, _, _) -> no. - -fix_block(Is, 0) -> - reverse(Is); -fix_block(Is, Words) -> - reverse(fix_block_1(Is, Words)). - -fix_block_1([{set,[],[],{alloc,Live,{F1,F2,Needed0,F3}}}|Is], Words) -> - case Needed0 - Words of - 0 -> - Is; - Needed -> - true = Needed >= 0, %Assertion. - [{set,[],[],{alloc,Live,{F1,F2,Needed,F3}}}|Is] - end; -fix_block_1([I|Is], Words) -> - [I|fix_block_1(Is, Words)]; -fix_block_1([], _Words) -> - %% Rare. The heap allocation was probably done by a binary - %% construction instruction. - []. - -dig_out_fc(Arity, Is0) -> - Regs0 = maps:from_list([{{x,X},{arg,X}} || X <- seq(0, Arity-1)]), - {Is,Acc0} = splitwith(fun({label,_}) -> false; - ({test,_,_,_}) -> false; - (_) -> true - end, Is0), - {Regs,Acc} = dig_out_fc_1(reverse(Is), Arity, Regs0, Acc0), - case Regs of - #{{x,0}:={atom,function_clause},{x,1}:=Args} -> - case moves_from_stack(Args, 0, []) of - {Moves,Arity} -> - {yes,function_clause,reverse(Moves, Acc)}; - {_,_} -> - no - end; - #{} -> - no - end. - -dig_out_fc_1([{block,Bl}|Is], Arity, Regs0, Acc) -> - Regs = dig_out_fc_block(Bl, Regs0), - dig_out_fc_1(Is, Arity, Regs, Acc); -dig_out_fc_1([{bs_set_position,_,_}=I|Is], Arity, Regs, Acc) -> - dig_out_fc_1(Is, Arity, Regs, [I|Acc]); -dig_out_fc_1([{bs_get_tail,Src,Dst,Live0}|Is], Arity, Regs0, Acc) -> - case Src of - {x,X} when X < Arity -> - %% The heuristic for determining the number of live - %% registers is likely to give an incorrect result. - %% Give up. - {#{},[]}; - _ -> - Regs = prune_xregs(Live0, Regs0), - Live = dig_out_stack_live(Regs, Live0), - I = {bs_get_tail,Src,Dst,Live}, - dig_out_fc_1(Is, Arity, Regs, [I|Acc]) - end; -dig_out_fc_1([_|_], _Arity, _Regs, _Acc) -> - {#{},[]}; -dig_out_fc_1([], _Arity, Regs, Acc) -> - {Regs,Acc}. - -dig_out_fc_block([{set,[],[],{alloc,Live,_}}|Is], Regs0) -> - Regs = prune_xregs(Live, Regs0), - dig_out_fc_block(Is, Regs); -dig_out_fc_block([{set,[Dst],[Hd,Tl],put_list}|Is], Regs0) -> - Regs = Regs0#{Dst=>{cons,get_reg(Hd, Regs0),get_reg(Tl, Regs0)}}, - dig_out_fc_block(Is, Regs); -dig_out_fc_block([{set,[Dst],[Src],move}|Is], Regs0) -> - Regs = Regs0#{Dst=>get_reg(Src, Regs0)}, - dig_out_fc_block(Is, Regs); -dig_out_fc_block([{set,_,_,_}|_], _Regs) -> - %% Unknown instruction. Fail. - #{}; -dig_out_fc_block([], Regs) -> Regs. - -dig_out_stack_live(Regs, Default) -> - Reg = {x,2}, - case Regs of - #{Reg:=List} -> - dig_out_stack_live_1(List, Default); - #{} -> - Default - end. - -dig_out_stack_live_1({cons,{arg,N},T}, Live) -> - dig_out_stack_live_1(T, max(N + 1, Live)); -dig_out_stack_live_1({cons,_,T}, Live) -> - dig_out_stack_live_1(T, Live); -dig_out_stack_live_1(nil, Live) -> - Live; -dig_out_stack_live_1(_, Live) -> Live. - -prune_xregs(Live, Regs) -> - maps:filter(fun({x,X}, _) -> X < Live end, Regs). - -moves_from_stack({cons,{arg,N},_}, I, _Acc) when N =/= I -> - %% Wrong argument. Give up. - {[],-1}; -moves_from_stack({cons,H,T}, I, Acc) -> - case H of - {arg,I} -> - moves_from_stack(T, I+1, Acc); - _ -> - moves_from_stack(T, I+1, [{move,H,{x,I}}|Acc]) - end; -moves_from_stack(nil, I, Acc) -> - {reverse(Acc),I}; -moves_from_stack({literal,[H|T]}, I, Acc) -> - Cons = {cons,tag_literal(H),tag_literal(T)}, - moves_from_stack(Cons, I, Acc); -moves_from_stack(_, _, _) -> - %% Not understood. Give up. - {[],-1}. - - -get_reg(R, Regs) -> - case Regs of - #{R:=Val} -> Val; - #{} -> R - end. - -tag_literal([]) -> nil; -tag_literal(T) when is_atom(T) -> {atom,T}; -tag_literal(T) when is_float(T) -> {float,T}; -tag_literal(T) when is_integer(T) -> {integer,T}; -tag_literal(T) -> {literal,T}. diff --git a/lib/compiler/src/beam_jump.erl b/lib/compiler/src/beam_jump.erl index 74f80ca70e..85fd89bbe8 100644 --- a/lib/compiler/src/beam_jump.erl +++ b/lib/compiler/src/beam_jump.erl @@ -580,8 +580,6 @@ is_unreachable_after(I) -> is_exit_instruction(I). -spec is_exit_instruction(instruction()) -> boolean(). -is_exit_instruction({call_ext,_,{extfunc,M,F,A}}) -> - erl_bifs:is_exit_bif(M, F, A); is_exit_instruction(if_end) -> true; is_exit_instruction({case_end,_}) -> true; is_exit_instruction({try_case_end,_}) -> true; diff --git a/lib/compiler/src/beam_kernel_to_ssa.erl b/lib/compiler/src/beam_kernel_to_ssa.erl index df95749fb3..2406a634e6 100644 --- a/lib/compiler/src/beam_kernel_to_ssa.erl +++ b/lib/compiler/src/beam_kernel_to_ssa.erl @@ -34,7 +34,7 @@ -type label() :: beam_ssa:label(). %% Main codegen structure. --record(cg, {lcount=1 :: label(), %Label counter +-record(cg, {lcount=1 :: label(), %Label counter bfail=1 :: label(), catch_label=none :: 'none' | label(), vars=#{} :: map(), %Defined variables. @@ -83,6 +83,7 @@ function(#k_fdef{anno=Anno0,func=Name,arity=Arity, cg_fun(Ke, St0) -> {UltimateFail,FailIs,St1} = make_failure(badarg, St0), + ?EXCEPTION_BLOCK = UltimateFail, %Assertion. St2 = St1#cg{bfail=UltimateFail,ultimate_failure=UltimateFail}, {B,St} = cg(Ke, St2), Asm = [{label,0}|B++FailIs], @@ -279,7 +280,7 @@ select_binary(#k_val_clause{val=#k_binary{segs=#k_var{name=Ctx0}},body=B}, #k_var{}=Src, Tf, Vf, St0) -> {Ctx,St1} = new_ssa_var(Ctx0, St0), {Bis0,St2} = match_cg(B, Vf, St1), - {TestIs,St} = make_cond_branch(succeeded, [Ctx], Tf, St2), + {TestIs,St} = make_succeeded(Ctx, {guard, Tf}, St2), Bis1 = [#b_set{op=bs_start_match,dst=Ctx, args=[ssa_arg(Src, St)]}] ++ TestIs ++ Bis0, Bis = finish_bs_matching(Bis1), @@ -311,6 +312,35 @@ make_cond_branch(Cond, Args, Fail, St0) -> make_uncond_branch(Fail) -> #b_br{bool=#b_literal{val=true},succ=Fail,fail=Fail}. +%% +%% The 'succeeded' instruction needs special treatment in catch blocks to +%% prevent the checked operation from being optimized away if a later pass +%% determines that it always fails. +%% + +make_succeeded(Var, {in_catch, CatchLbl}, St0) -> + {Bool, St1} = new_ssa_var('@ssa_bool', St0), + {Succ, St2} = new_label(St1), + {Fail, St} = new_label(St2), + + Check = [#b_set{op=succeeded,dst=Bool,args=[Var]}, + #b_br{bool=Bool,succ=Succ,fail=Fail}], + + %% Add a dummy block that references the checked variable, ensuring it + %% stays alive and that it won't be merged with the landing pad. + Trampoline = [{label,Fail}, + #b_set{op=exception_trampoline,args=[Var]}, + make_uncond_branch(CatchLbl)], + + {Check ++ Trampoline ++ [{label,Succ}], St}; +make_succeeded(Var, {no_catch, Fail}, St) -> + %% Ultimate failure raises an exception, so we must treat it as if it were + %% in a catch to keep it from being optimized out. + #cg{ultimate_failure=Fail} = St, %Assertion + make_succeeded(Var, {in_catch, Fail}, St); +make_succeeded(Var, {guard, Fail}, St) -> + make_cond_branch(succeeded, [Var], Fail, St). + %% Instructions for selection of binary segments. select_bin_segs(Scs, Ivar, Tf, St) -> @@ -394,7 +424,7 @@ select_extract_int(#k_var{name=Tl}, Val, #k_int{val=Sz}, U, Fs, Vf, <<Val:Bits/little>> end, Bits = bit_size(Bin), %Assertion. - {TestIs,St} = make_cond_branch(succeeded, [Dst], Vf, St1), + {TestIs,St} = make_succeeded(Dst, {guard, Vf}, St1), Set = #b_set{op=bs_match,dst=Dst, args=[#b_literal{val=string},Ctx,#b_literal{val=Bin}]}, {[Set|TestIs],St}. @@ -412,7 +442,7 @@ build_bs_instr(Anno, Type, Fail, Ctx, Size, Unit0, Flags0, Dst, St0) -> #b_set{anno=Anno,op=bs_match,dst=Dst, args=[TypeArg,Ctx,Flags]} end, - {Is,St} = make_cond_branch(succeeded, [Dst], Fail, St0), + {Is,St} = make_succeeded(Dst, {guard, Fail}, St0), {[Get|Is],St}. select_val(#k_val_clause{val=#k_tuple{es=Es},body=B}, V, Vf, St0) -> @@ -475,7 +505,7 @@ select_extract_map([P|Ps], Src, Fail, St0) -> Key = ssa_arg(Key0, St0), {Dst,St1} = new_ssa_var(Dst0, St0), Set = #b_set{op=get_map_element,dst=Dst,args=[MapSrc,Key]}, - {TestIs,St2} = make_cond_branch(succeeded, [Dst], Fail, St1), + {TestIs,St2} = make_succeeded(Dst, {guard, Fail}, St1), {Is,St} = select_extract_map(Ps, Src, Fail, St2), {[Set|TestIs]++Is,St}; select_extract_map([], _, _, St) -> @@ -596,7 +626,7 @@ match_fmf(F, LastFail, St0, [H|T]) -> {Rs,St3} = match_fmf(F, LastFail, St2, T), {R ++ [{label,Fail}] ++ Rs,St3}. -%% fail_label(State) -> {Where,FailureLabel}. +%% fail_context(State) -> {Where,FailureLabel}. %% Where = guard | no_catch | in_catch %% Return an indication of which part of a function code is %% being generated for and the appropriate failure label to @@ -609,7 +639,7 @@ match_fmf(F, LastFail, St0, [H|T]) -> %% a try/catch or catch. %% in_catch - In the scope of a try/catch or catch. -fail_label(#cg{catch_label=Catch,bfail=Fail,ultimate_failure=Ult}) -> +fail_context(#cg{catch_label=Catch,bfail=Fail,ultimate_failure=Ult}) -> if Fail =/= Ult -> {guard,Fail}; @@ -619,14 +649,6 @@ fail_label(#cg{catch_label=Catch,bfail=Fail,ultimate_failure=Ult}) -> {in_catch,Catch} end. -%% bif_fail_label(State) -> FailureLabel. -%% Return the appropriate failure label for a guard BIF call or -%% primop that fails. - -bif_fail_label(St) -> - {_,Fail} = fail_label(St), - Fail. - %% call_cg(Func, [Arg], [Ret], Le, State) -> %% {[Ainstr],State}. %% enter_cg(Func, [Arg], Le, St) -> {[Ainstr],St}. @@ -635,7 +657,7 @@ bif_fail_label(St) -> call_cg(Func, As, [], Le, St) -> call_cg(Func, As, [#k_var{name='@ssa_ignored'}], Le, St); call_cg(Func0, As, [#k_var{name=R}|MoreRs]=Rs, Le, St0) -> - case fail_label(St0) of + case fail_context(St0) of {guard,Fail} -> %% Inside a guard. The only allowed function call is to %% erlang:error/1,2. We will generate a branch to the @@ -645,7 +667,7 @@ call_cg(Func0, As, [#k_var{name=R}|MoreRs]=Rs, Le, St0) -> [#k_var{name=DestVar}] = Rs, St = set_ssa_var(DestVar, #b_literal{val=unused}, St0), {[make_uncond_branch(Fail),#cg_unreachable{}],St}; - {Catch,Fail} -> + FailCtx -> %% Ordinary function call in a function body. Args = ssa_args(As, St0), {Ret,St1} = new_ssa_var(R, St0), @@ -657,13 +679,9 @@ call_cg(Func0, As, [#k_var{name=R}|MoreRs]=Rs, Le, St0) -> St2 = foldl(fun(#k_var{name=Dummy}, S) -> set_ssa_var(Dummy, #b_literal{val=unused}, S) end, St1, MoreRs), - case Catch of - no_catch -> - {[Call],St2}; - in_catch -> - {TestIs,St} = make_cond_branch(succeeded, [Ret], Fail, St2), - {[Call|TestIs],St} - end + + {TestIs,St} = make_succeeded(Ret, FailCtx, St2), + {[Call|TestIs],St} end. enter_cg(Func0, As0, Le, St0) -> @@ -748,8 +766,8 @@ bif_cg(Bif, As0, [#k_var{name=Dst0}], Le, St0) -> I = #b_set{anno=line_anno(Le),op={bif,Bif},dst=Dst,args=As}, case erl_bifs:is_safe(erlang, Bif, length(As)) of false -> - Fail = bif_fail_label(St1), - {Is,St} = make_cond_branch(succeeded, [Dst], Fail, St1), + FailCtx = fail_context(St1), + {Is,St} = make_succeeded(Dst, FailCtx, St1), {[I|Is],St}; true-> {[I],St1} @@ -797,7 +815,7 @@ cg_recv_mesg(#k_var{name=R}, Rm, Tl, Le, St0) -> {Dst,St1} = new_ssa_var(R, St0), {Mis,St2} = match_cg(Rm, none, St1), RecvLbl = St1#cg.recv, - {TestIs,St} = make_cond_branch(succeeded, [Dst], Tl, St2), + {TestIs,St} = make_succeeded(Dst, {guard, Tl}, St2), Is = [#b_br{anno=line_anno(Le),bool=#b_literal{val=true}, succ=RecvLbl,fail=RecvLbl}, {label,RecvLbl}, @@ -813,7 +831,7 @@ cg_recv_wait(Te, Es, St0) -> {Tis,St1} = cg(Es, St0), Args = [ssa_arg(Te, St1)], {WaitDst,St2} = new_ssa_var('@ssa_wait', St1), - {WaitIs,St} = make_cond_branch(succeeded, [WaitDst], St1#cg.recv, St2), + {WaitIs,St} = make_succeeded(WaitDst, {guard, St1#cg.recv}, St2), %% Infinite timeout will be optimized later. Is = [#b_set{op=wait_timeout,dst=WaitDst,args=Args}] ++ WaitIs ++ [#b_set{op=timeout}] ++ Tis, @@ -924,9 +942,9 @@ put_cg([#k_var{name=R}], #k_tuple{es=Es}, _Le, St0) -> PutTuple = #b_set{op=put_tuple,dst=Ret,args=Args}, {[PutTuple],St}; put_cg([#k_var{name=R}], #k_binary{segs=Segs}, Le, St0) -> - Fail = bif_fail_label(St0), + FailCtx = fail_context(St0), {Dst,St1} = new_ssa_var(R, St0), - cg_binary(Dst, Segs, Fail, Le, St1); + cg_binary(Dst, Segs, FailCtx, Le, St1); put_cg([#k_var{name=R}], #k_map{op=Op,var=Map, es=[#k_map_pair{key=#k_var{}=K,val=V}]}, Le, St0) -> @@ -955,14 +973,14 @@ put_cg([#k_var{name=R}], Con0, _Le, St0) -> {[],St}. put_cg_map(LineAnno, Op, SrcMap, Dst, List, St0) -> - Fail = bif_fail_label(St0), Args = [#b_literal{val=Op},SrcMap|List], PutMap = #b_set{anno=LineAnno,op=put_map,dst=Dst,args=Args}, if Op =:= assoc -> {[PutMap],St0}; true -> - {Is,St} = make_cond_branch(succeeded, [Dst], Fail, St0), + FailCtx = fail_context(St0), + {Is,St} = make_succeeded(Dst, FailCtx, St0), {[PutMap|Is],St} end. @@ -970,8 +988,8 @@ put_cg_map(LineAnno, Op, SrcMap, Dst, List, St0) -> %%% Code generation for constructing binaries. %%% -cg_binary(Dst, Segs0, Fail, Le, St0) -> - {PutCode0,SzCalc0,St1} = cg_bin_put(Segs0, Fail, St0), +cg_binary(Dst, Segs0, FailCtx, Le, St0) -> + {PutCode0,SzCalc0,St1} = cg_bin_put(Segs0, FailCtx, St0), LineAnno = line_anno(Le), Anno = Le#k.a, case PutCode0 of @@ -980,8 +998,8 @@ cg_binary(Dst, Segs0, Fail, Le, St0) -> {label,_}|_] -> #k_bin_seg{unit=Unit0,next=Segs} = Segs0, Unit = #b_literal{val=Unit0}, - {PutCode,SzCalc1,St2} = cg_bin_put(Segs, Fail, St1), - {_,SzVar,SzCode0,St3} = cg_size_calc(1, SzCalc1, Fail, St2), + {PutCode,SzCalc1,St2} = cg_bin_put(Segs, FailCtx, St1), + {_,SzVar,SzCode0,St3} = cg_size_calc(1, SzCalc1, FailCtx, St2), SzCode = cg_bin_anno(SzCode0, LineAnno), Args = case member(single_use, Anno) of true -> @@ -990,14 +1008,14 @@ cg_binary(Dst, Segs0, Fail, Le, St0) -> [#b_literal{val=append},Src,SzVar,Unit] end, BsInit = #b_set{anno=LineAnno,op=bs_init,dst=Dst,args=Args}, - {TestIs,St} = make_cond_branch(succeeded, [Dst], Fail, St3), + {TestIs,St} = make_succeeded(Dst, FailCtx, St3), {SzCode ++ [BsInit] ++ TestIs ++ PutCode,St}; [#b_set{op=bs_put}|_] -> - {Unit,SzVar,SzCode0,St2} = cg_size_calc(8, SzCalc0, Fail, St1), + {Unit,SzVar,SzCode0,St2} = cg_size_calc(8, SzCalc0, FailCtx, St1), SzCode = cg_bin_anno(SzCode0, LineAnno), Args = [#b_literal{val=new},SzVar,Unit], BsInit = #b_set{anno=LineAnno,op=bs_init,dst=Dst,args=Args}, - {TestIs,St} = make_cond_branch(succeeded, [Dst], Fail, St2), + {TestIs,St} = make_succeeded(Dst, FailCtx, St2), {SzCode ++ [BsInit] ++ TestIs ++ PutCode0,St} end. @@ -1005,18 +1023,18 @@ cg_bin_anno([Set|Sets], Anno) -> [Set#b_set{anno=Anno}|Sets]; cg_bin_anno([], _) -> []. -%% cg_size_calc(PreferredUnit, SzCalc, Fail, St0) -> +%% cg_size_calc(PreferredUnit, SzCalc, FailCtx, St0) -> %% {ActualUnit,SizeVariable,SizeCode,St}. %% Generate size calculation code. -cg_size_calc(Unit, error, _Fail, St) -> +cg_size_calc(Unit, error, _FailCtx, St) -> {#b_literal{val=Unit},#b_literal{val=badarg},[],St}; -cg_size_calc(8, [{1,_}|_]=SzCalc, Fail, St) -> - cg_size_calc(1, SzCalc, Fail, St); -cg_size_calc(8, SzCalc, Fail, St0) -> - {Var,Pre,St} = cg_size_calc_1(SzCalc, Fail, St0), +cg_size_calc(8, [{1,_}|_]=SzCalc, FailCtx, St) -> + cg_size_calc(1, SzCalc, FailCtx, St); +cg_size_calc(8, SzCalc, FailCtx, St0) -> + {Var,Pre,St} = cg_size_calc_1(SzCalc, FailCtx, St0), {#b_literal{val=8},Var,Pre,St}; -cg_size_calc(1, SzCalc0, Fail, St0) -> +cg_size_calc(1, SzCalc0, FailCtx, St0) -> SzCalc = map(fun({8,#b_literal{val=Size}}) -> {1,#b_literal{val=8*Size}}; ({8,{{bif,byte_size},Src}}) -> @@ -1026,54 +1044,54 @@ cg_size_calc(1, SzCalc0, Fail, St0) -> ({_,_}=Pair) -> Pair end, SzCalc0), - {Var,Pre,St} = cg_size_calc_1(SzCalc, Fail, St0), + {Var,Pre,St} = cg_size_calc_1(SzCalc, FailCtx, St0), {#b_literal{val=1},Var,Pre,St}. -cg_size_calc_1(SzCalc, Fail, St0) -> - cg_size_calc_2(SzCalc, #b_literal{val=0}, Fail, St0). +cg_size_calc_1(SzCalc, FailCtx, St0) -> + cg_size_calc_2(SzCalc, #b_literal{val=0}, FailCtx, St0). -cg_size_calc_2([{_,{'*',Unit,{_,_}=Bif}}|T], Sum0, Fail, St0) -> - {Sum1,Pre0,St1} = cg_size_calc_2(T, Sum0, Fail, St0), - {BifDst,Pre1,St2} = cg_size_bif(Bif, Fail, St1), - {Sum,Pre2,St} = cg_size_add(Sum1, BifDst, Unit, Fail, St2), +cg_size_calc_2([{_,{'*',Unit,{_,_}=Bif}}|T], Sum0, FailCtx, St0) -> + {Sum1,Pre0,St1} = cg_size_calc_2(T, Sum0, FailCtx, St0), + {BifDst,Pre1,St2} = cg_size_bif(Bif, FailCtx, St1), + {Sum,Pre2,St} = cg_size_add(Sum1, BifDst, Unit, FailCtx, St2), {Sum,Pre0++Pre1++Pre2,St}; -cg_size_calc_2([{_,#b_literal{}=Sz}|T], Sum0, Fail, St0) -> - {Sum1,Pre0,St1} = cg_size_calc_2(T, Sum0, Fail, St0), - {Sum,Pre,St} = cg_size_add(Sum1, Sz, #b_literal{val=1}, Fail, St1), +cg_size_calc_2([{_,#b_literal{}=Sz}|T], Sum0, FailCtx, St0) -> + {Sum1,Pre0,St1} = cg_size_calc_2(T, Sum0, FailCtx, St0), + {Sum,Pre,St} = cg_size_add(Sum1, Sz, #b_literal{val=1}, FailCtx, St1), {Sum,Pre0++Pre,St}; -cg_size_calc_2([{_,#b_var{}=Sz}|T], Sum0, Fail, St0) -> - {Sum1,Pre0,St1} = cg_size_calc_2(T, Sum0, Fail, St0), - {Sum,Pre,St} = cg_size_add(Sum1, Sz, #b_literal{val=1}, Fail, St1), +cg_size_calc_2([{_,#b_var{}=Sz}|T], Sum0, FailCtx, St0) -> + {Sum1,Pre0,St1} = cg_size_calc_2(T, Sum0, FailCtx, St0), + {Sum,Pre,St} = cg_size_add(Sum1, Sz, #b_literal{val=1}, FailCtx, St1), {Sum,Pre0++Pre,St}; -cg_size_calc_2([{_,{_,_}=Bif}|T], Sum0, Fail, St0) -> - {Sum1,Pre0,St1} = cg_size_calc_2(T, Sum0, Fail, St0), - {BifDst,Pre1,St2} = cg_size_bif(Bif, Fail, St1), - {Sum,Pre2,St} = cg_size_add(Sum1, BifDst, #b_literal{val=1}, Fail, St2), +cg_size_calc_2([{_,{_,_}=Bif}|T], Sum0, FailCtx, St0) -> + {Sum1,Pre0,St1} = cg_size_calc_2(T, Sum0, FailCtx, St0), + {BifDst,Pre1,St2} = cg_size_bif(Bif, FailCtx, St1), + {Sum,Pre2,St} = cg_size_add(Sum1, BifDst, #b_literal{val=1}, FailCtx, St2), {Sum,Pre0++Pre1++Pre2,St}; -cg_size_calc_2([], Sum, _Fail, St) -> +cg_size_calc_2([], Sum, _FailCtx, St) -> {Sum,[],St}. -cg_size_bif(#b_var{}=Var, _Fail, St) -> +cg_size_bif(#b_var{}=Var, _FailCtx, St) -> {Var,[],St}; -cg_size_bif({Name,Src}, Fail, St0) -> +cg_size_bif({Name,Src}, FailCtx, St0) -> {Dst,St1} = new_ssa_var('@ssa_bif', St0), Bif = #b_set{op=Name,dst=Dst,args=[Src]}, - {TestIs,St} = make_cond_branch(succeeded, [Dst], Fail, St1), + {TestIs,St} = make_succeeded(Dst, FailCtx, St1), {Dst,[Bif|TestIs],St}. -cg_size_add(#b_literal{val=0}, Val, #b_literal{val=1}, _Fail, St) -> +cg_size_add(#b_literal{val=0}, Val, #b_literal{val=1}, _FailCtx, St) -> {Val,[],St}; -cg_size_add(A, B, Unit, Fail, St0) -> +cg_size_add(A, B, Unit, FailCtx, St0) -> {Dst,St1} = new_ssa_var('@ssa_sum', St0), - {TestIs,St} = make_cond_branch(succeeded, [Dst], Fail, St1), + {TestIs,St} = make_succeeded(Dst, FailCtx, St1), BsAdd = #b_set{op=bs_add,dst=Dst,args=[A,B,Unit]}, {Dst,[BsAdd|TestIs],St}. -cg_bin_put(Seg, Fail, St) -> - cg_bin_put_1(Seg, Fail, [], [], St). +cg_bin_put(Seg, FailCtx, St) -> + cg_bin_put_1(Seg, FailCtx, [], [], St). cg_bin_put_1(#k_bin_seg{size=Size0,unit=U,type=T,flags=Fs,seg=Src0,next=Next}, - Fail, Acc, SzCalcAcc, St0) -> + FailCtx, Acc, SzCalcAcc, St0) -> [Src,Size] = ssa_args([Src0,Size0], St0), NeedSize = bs_need_size(T), TypeArg = #b_literal{val=T}, @@ -1083,9 +1101,12 @@ cg_bin_put_1(#k_bin_seg{size=Size0,unit=U,type=T,flags=Fs,seg=Src0,next=Next}, true -> [TypeArg,Flags,Src,Size,Unit]; false -> [TypeArg,Flags,Src] end, - {Is,St} = make_cond_branch(bs_put, Args, Fail, St0), + %% bs_put has its own 'succeeded' logic, and should always jump directly to + %% the fail label regardless of whether it's in a catch or not. + {_, FailLbl} = FailCtx, + {Is,St} = make_cond_branch(bs_put, Args, FailLbl, St0), SzCalc = bin_size_calc(T, Src, Size, U), - cg_bin_put_1(Next, Fail, reverse(Is, Acc), [SzCalc|SzCalcAcc], St); + cg_bin_put_1(Next, FailCtx, reverse(Is, Acc), [SzCalc|SzCalcAcc], St); cg_bin_put_1(#k_bin_end{}, _, Acc, SzCalcAcc, St) -> SzCalc = fold_size_calc(SzCalcAcc, 0, []), {reverse(Acc),SzCalc,St}. diff --git a/lib/compiler/src/beam_ssa.erl b/lib/compiler/src/beam_ssa.erl index 6492d1e1bf..77619368c7 100644 --- a/lib/compiler/src/beam_ssa.erl +++ b/lib/compiler/src/beam_ssa.erl @@ -21,7 +21,7 @@ -module(beam_ssa). -export([add_anno/3,get_anno/2,get_anno/3, - clobbers_xregs/1,def/2,def_used/2, + clobbers_xregs/1,def/2,def_unused/3, definitions/1, dominators/1,common_dominators/3, flatmapfold_instrs_rpo/4, @@ -79,7 +79,7 @@ -type var_base() :: atom() | non_neg_integer(). -type literal_value() :: atom() | integer() | float() | list() | - nil() | tuple() | map() | binary(). + nil() | tuple() | map() | binary() | fun(). -type op() :: {'bif',atom()} | {'float',float_op()} | prim_op() | cg_prim_op(). -type anno() :: #{atom() := any()}. @@ -101,7 +101,7 @@ 'bs_match' | 'bs_put' | 'bs_start_match' | 'bs_test_tail' | 'bs_utf16_size' | 'bs_utf8_size' | 'build_stacktrace' | 'call' | 'catch_end' | - 'extract' | + 'extract' | 'exception_trampoline' | 'get_hd' | 'get_map_element' | 'get_tl' | 'get_tuple_element' | 'has_map_field' | 'is_nonempty_list' | 'is_tagged_tuple' | @@ -120,10 +120,11 @@ %% Primops only used internally during code generation. -type cg_prim_op() :: 'bs_get' | 'bs_get_position' | 'bs_match_string' | 'bs_restore' | 'bs_save' | 'bs_set_position' | 'bs_skip' | - 'copy' | 'put_tuple_arity' | 'put_tuple_element' | - 'put_tuple_elements' | 'set_tuple_element'. + 'copy' | 'match_fail' | 'put_tuple_arity' | + 'put_tuple_element' | 'put_tuple_elements' | + 'set_tuple_element'. --import(lists, [foldl/3,keyfind/3,mapfoldl/3,member/2,reverse/1,umerge/1]). +-import(lists, [foldl/3,keyfind/3,mapfoldl/3,member/2,reverse/1]). -spec add_anno(Key, Value, Construct) -> Construct when Key :: atom(), @@ -319,17 +320,18 @@ def(Ls, Blocks) -> Blks = [map_get(L, Blocks) || L <- Top], def_1(Blks, []). --spec def_used(Ls, Blocks) -> {Def,Used} when +-spec def_unused(Ls, Used, Blocks) -> {Def,Unused} when Ls :: [label()], + Used :: ordsets:ordset(var_name()), Blocks :: block_map(), Def :: ordsets:ordset(var_name()), - Used :: ordsets:ordset(var_name()). + Unused :: ordsets:ordset(var_name()). -def_used(Ls, Blocks) -> +def_unused(Ls, Unused, Blocks) -> Top = rpo(Ls, Blocks), Blks = [map_get(L, Blocks) || L <- Top], Preds = cerl_sets:from_list(Top), - def_used_1(Blks, Preds, [], []). + def_unused_1(Blks, Preds, [], Unused). %% dominators(BlockMap) -> {Dominators,Numbering}. %% Calculate the dominator tree, returning a map where each entry @@ -651,34 +653,28 @@ is_commutative('=/=') -> true; is_commutative('/=') -> true; is_commutative(_) -> false. -def_used_1([#b_blk{is=Is,last=Last}|Bs], Preds, Def0, UsedAcc) -> - {Def,Used} = def_used_is(Is, Preds, Def0, used(Last)), - case Used of - [] -> - def_used_1(Bs, Preds, Def, UsedAcc); - [_|_] -> - def_used_1(Bs, Preds, Def, [Used|UsedAcc]) - end; -def_used_1([], _Preds, Def0, UsedAcc) -> - Def = ordsets:from_list(Def0), - Used = umerge(UsedAcc), - {Def,Used}. +def_unused_1([#b_blk{is=Is,last=Last}|Bs], Preds, Def0, Unused0) -> + Unused1 = ordsets:subtract(Unused0, used(Last)), + {Def,Unused} = def_unused_is(Is, Preds, Def0, Unused1), + def_unused_1(Bs, Preds, Def, Unused); +def_unused_1([], _Preds, Def, Unused) -> + {ordsets:from_list(Def), Unused}. -def_used_is([#b_set{op=phi,dst=Dst,args=Args}|Is], - Preds, Def0, Used0) -> +def_unused_is([#b_set{op=phi,dst=Dst,args=Args}|Is], + Preds, Def0, Unused0) -> Def = [Dst|Def0], %% We must be careful to only include variables that will %% be used when arriving from one of the predecessor blocks %% in Preds. - Used1 = [V || {#b_var{}=V,L} <- Args, cerl_sets:is_element(L, Preds)], - Used = ordsets:union(ordsets:from_list(Used1), Used0), - def_used_is(Is, Preds, Def, Used); -def_used_is([#b_set{dst=Dst}=I|Is], Preds, Def0, Used0) -> + Unused1 = [V || {#b_var{}=V,L} <- Args, cerl_sets:is_element(L, Preds)], + Unused = ordsets:subtract(Unused0, ordsets:from_list(Unused1)), + def_unused_is(Is, Preds, Def, Unused); +def_unused_is([#b_set{dst=Dst}=I|Is], Preds, Def0, Unused0) -> Def = [Dst|Def0], - Used = ordsets:union(used(I), Used0), - def_used_is(Is, Preds, Def, Used); -def_used_is([], _Preds, Def, Used) -> - {Def,Used}. + Unused = ordsets:subtract(Unused0, used(I)), + def_unused_is(Is, Preds, Def, Unused); +def_unused_is([], _Preds, Def, Unused) -> + {Def,Unused}. def_1([#b_blk{is=Is}|Bs], Def0) -> Def = def_is(Is, Def0), diff --git a/lib/compiler/src/beam_ssa.hrl b/lib/compiler/src/beam_ssa.hrl index fa76b08453..509a94135e 100644 --- a/lib/compiler/src/beam_ssa.hrl +++ b/lib/compiler/src/beam_ssa.hrl @@ -62,5 +62,13 @@ -record(b_local, {name :: beam_ssa:b_literal(), arity :: non_neg_integer()}). -%% If this block exists, it calls erlang:error(badarg). --define(BADARG_BLOCK, 1). +%% This is a psuedo-block used to express that certain instructions and BIFs +%% throw exceptions on failure. The code generator rewrites all branches to +%% this block to {f,0} which causes the instruction to throw an exception +%% instead of branching. +%% +%% Since this is not an ordinary block, it's illegal to merge it with other +%% blocks, and jumps are only valid when we know that an exception will be +%% thrown by the operation that branches here; the *block itself* does not +%% throw an exception. +-define(EXCEPTION_BLOCK, 1). diff --git a/lib/compiler/src/beam_ssa_bsm.erl b/lib/compiler/src/beam_ssa_bsm.erl index 1ac9e6a3bb..7a8dc127d7 100644 --- a/lib/compiler/src/beam_ssa_bsm.erl +++ b/lib/compiler/src/beam_ssa_bsm.erl @@ -57,6 +57,7 @@ -export([module/2, format_error/1]). -include("beam_ssa.hrl"). +-include("beam_types.hrl"). -import(lists, [member/2, reverse/1, splitwith/2, map/2, foldl/3, mapfoldl/3, nth/2, max/1, unzip/1]). @@ -683,9 +684,9 @@ aca_copy_successors(Lbl0, Blocks0, Counter0) -> Lbl = maps:get(Lbl0, BRs), {Lbl, Blocks, Counter}. -aca_cs_build_brs([?BADARG_BLOCK=Lbl | Path], Counter, Acc) -> - %% ?BADARG_BLOCK is a marker and not an actual block, so renaming it will - %% break exception handling. +aca_cs_build_brs([?EXCEPTION_BLOCK=Lbl | Path], Counter, Acc) -> + %% ?EXCEPTION_BLOCK is a marker and not an actual block, so renaming it + %% will break exception handling. aca_cs_build_brs(Path, Counter, Acc#{ Lbl => Lbl }); aca_cs_build_brs([Lbl | Path], Counter0, Acc) -> aca_cs_build_brs(Path, Counter0 + 1, Acc#{ Lbl => Counter0 }); @@ -883,8 +884,7 @@ annotate_context_parameters(F, ModInfo) -> %% Assertion. error(conflicting_parameter_types); (K, suitable_for_reuse, Acc) -> - T = beam_validator:type_anno(match_context), - Acc#{ K => T }; + Acc#{ K => #t_bs_context{} }; (_K, _V, Acc) -> Acc end, TypeAnno0, ParamInfo), diff --git a/lib/compiler/src/beam_ssa_codegen.erl b/lib/compiler/src/beam_ssa_codegen.erl index 08641e2abc..ff880c6296 100644 --- a/lib/compiler/src/beam_ssa_codegen.erl +++ b/lib/compiler/src/beam_ssa_codegen.erl @@ -28,7 +28,7 @@ -include("beam_ssa.hrl"). --import(lists, [foldl/3,keymember/3,keysort/2,last/1,map/2,mapfoldl/3, +-import(lists, [foldl/3,keymember/3,keysort/2,map/2,mapfoldl/3, reverse/1,reverse/2,sort/1,splitwith/2,takewhile/2]). -record(cg, {lcount=1 :: beam_label(), %Label counter @@ -37,7 +37,8 @@ used_labels=gb_sets:empty() :: gb_sets:set(ssa_label()), regs=#{} :: #{beam_ssa:var_name()=>ssa_register()}, ultimate_fail=1 :: beam_label(), - catches=gb_sets:empty() :: gb_sets:set(ssa_label()) + catches=gb_sets:empty() :: gb_sets:set(ssa_label()), + fc_label=1 :: beam_label() }). -spec module(beam_ssa:b_module(), [compile:option()]) -> @@ -114,17 +115,17 @@ functions(Forms, AtomMod) -> function(#b_function{anno=Anno,bs=Blocks}, AtomMod, St0) -> #{func_info:={_,Name,Arity}} = Anno, try - assert_badarg_block(Blocks), %Assertion. + assert_exception_block(Blocks), %Assertion. Regs = maps:get(registers, Anno), St1 = St0#cg{labels=#{},used_labels=gb_sets:empty(), regs=Regs}, {Fi,St2} = new_label(St1), %FuncInfo label {Entry,St3} = local_func_label(Name, Arity, St2), {Ult,St4} = new_label(St3), %Ultimate failure - Labels = (St4#cg.labels)#{0=>Entry,?BADARG_BLOCK=>0}, + Labels = (St4#cg.labels)#{0=>Entry,?EXCEPTION_BLOCK=>0}, St5 = St4#cg{labels=Labels,used_labels=gb_sets:singleton(Entry), ultimate_fail=Ult}, - {Body,St} = cg_fun(Blocks, St5), + {Body,St} = cg_fun(Blocks, St5#cg{fc_label=Fi}), Asm = [{label,Fi},line(Anno), {func_info,AtomMod,{atom,Name},Arity}] ++ add_parameter_annos(Body, Anno) ++ @@ -137,10 +138,10 @@ function(#b_function{anno=Anno,bs=Blocks}, AtomMod, St0) -> erlang:raise(Class, Error, Stack) end. -assert_badarg_block(Blocks) -> - %% Assertion: ?BADARG_BLOCK must be the call erlang:error(badarg). +assert_exception_block(Blocks) -> + %% Assertion: ?EXCEPTION_BLOCK must be a call erlang:error(badarg). case Blocks of - #{?BADARG_BLOCK:=Blk} -> + #{?EXCEPTION_BLOCK:=Blk} -> #b_blk{is=[#b_set{op=call,dst=Ret, args=[#b_remote{mod=#b_literal{val=erlang}, name=#b_literal{val=error}}, @@ -148,7 +149,7 @@ assert_badarg_block(Blocks) -> last=#b_ret{arg=Ret}} = Blk, ok; #{} -> - %% ?BADARG_BLOCK has been removed because it was never used. + %% ?EXCEPTION_BLOCK has been removed because it was never used. ok end. @@ -384,6 +385,7 @@ classify_heap_need(is_tagged_tuple) -> neutral; classify_heap_need(kill_try_tag) -> gc; classify_heap_need(landingpad) -> gc; classify_heap_need(make_fun) -> gc; +classify_heap_need(match_fail) -> gc; classify_heap_need(new_try_tag) -> gc; classify_heap_need(peek_message) -> gc; classify_heap_need(put_map) -> gc; @@ -629,7 +631,7 @@ liveness_get(S, LiveMap) -> end. liveness_successors(Terminator) -> - successors(Terminator) -- [?BADARG_BLOCK]. + successors(Terminator) -- [?EXCEPTION_BLOCK]. liveness_is([#cg_alloc{}=I0|Is], Regs, Live, Acc) -> I = I0#cg_alloc{live=num_live(Live, Regs)}, @@ -973,6 +975,12 @@ cg_block(Is0, Last, Next, St0) -> case Last of #cg_br{succ=Next,fail=Next} -> cg_block(Is0, none, St0); + #cg_br{succ=Same,fail=Same} when Same =:= ?EXCEPTION_BLOCK -> + %% An expression in this block *always* throws an exception, so we + %% terminate it with an 'if_end' to make sure the validator knows + %% that the following instructions won't actually be reached. + {Is,St} = cg_block(Is0, none, St0), + {Is++[if_end],St}; #cg_br{succ=Same,fail=Same} -> {Fail,St1} = use_block_label(Same, St0), {Is,St} = cg_block(Is0, none, St1), @@ -1178,6 +1186,10 @@ cg_block([#cg_set{op=call}=I, #cg_set{op=succeeded,dst=Bool}], {Bool,_Fail}, St) -> %% A call in try/catch block. cg_block([I], none, St); +cg_block([#cg_set{op=match_fail}=I, + #cg_set{op=succeeded,dst=Bool}], {Bool,_Fail}, St) -> + %% A match_fail instruction in a try/catch block. + cg_block([I], none, St); cg_block([#cg_set{op=get_map_element,dst=Dst0,args=Args0}, #cg_set{op=succeeded,dst=Bool}], {Bool,Fail0}, St) -> [Dst,Map,Key] = beam_args([Dst0|Args0], St), @@ -1239,6 +1251,28 @@ cg_block([#cg_set{op=copy}|_]=T0, Context, St0) -> no -> {Is,St} end; +cg_block([#cg_set{op=match_fail,args=Args0,anno=Anno}], none, St) -> + Args = beam_args(Args0, St), + Is = cg_match_fail(Args, line(Anno), none), + {Is,St}; +cg_block([#cg_set{op=match_fail,args=Args0,anno=Anno}|T], Context, St0) -> + FcLabel = case Context of + {return,_,none} -> + %% There is no stack frame. If this is a function_clause + %% exception, it is safe to jump to the label of the + %% func_info instruction. + St0#cg.fc_label; + _ -> + %% This is most probably not a function_clause. + %% If this is a function_clause exception + %% (rare), it is not safe to jump to the + %% func_info label. + none + end, + Args = beam_args(Args0, St0), + Is0 = cg_match_fail(Args, line(Anno), FcLabel), + {Is1,St} = cg_block(T, Context, St0), + {Is0++Is1,St}; cg_block([#cg_set{op=Op,dst=Dst0,args=Args0}=Set], none, St) -> [Dst|Args] = beam_args([Dst0|Args0], St), Is = cg_instr(Op, Args, Dst, Set), @@ -1270,8 +1304,7 @@ cg_copy(T0, St) -> end, T0), Moves0 = cg_copy_1(Copies, St), Moves1 = [Move || {move,Src,Dst}=Move <- Moves0, Src =/= Dst], - Scratch = {x,1022}, - Moves = order_moves(Moves1, Scratch), + Moves = order_moves(Moves1), {Moves,T}. cg_copy_1([#cg_set{dst=Dst0,args=Args}|T], St) -> @@ -1512,6 +1545,42 @@ cg_call(#cg_set{anno=Anno,op=call,dst=Dst0,args=Args0}, Is = setup_args(Args++[Func], Anno, Context, St) ++ Line ++ Call, {Is,St}. +cg_match_fail([{atom,function_clause}|Args], Line, Fc) -> + case Fc of + none -> + %% There is a stack frame (probably because of inlining). + %% Jumping to the func_info label is not allowed by + %% beam_validator. Rewrite the instruction as a call to + %% erlang:error/2. + make_fc(Args, Line); + _ -> + setup_args(Args) ++ [{jump,{f,Fc}}] + end; +cg_match_fail([{atom,Op}], Line, _Fc) -> + [Line,Op]; +cg_match_fail([{atom,Op},Val], Line, _Fc) -> + [Line,{Op,Val}]. + +make_fc(Args, Line) -> + %% Recreate the original call to erlang:error/2. + Live = foldl(fun({x,X}, A) -> max(X+1, A); + (_, A) -> A + end, 0, Args), + TmpReg = {x,Live}, + StkMoves = build_stk(reverse(Args), TmpReg, nil), + [{test_heap,2*length(Args),Live}|StkMoves] ++ + [{move,{atom,function_clause},{x,0}}, + Line, + {call_ext,2,{extfunc,erlang,error,2}}]. + +build_stk([V], _TmpReg, Tail) -> + [{put_list,V,Tail,{x,1}}]; +build_stk([V|Vs], TmpReg, Tail) -> + I = {put_list,V,Tail,TmpReg}, + [I|build_stk(Vs, TmpReg, TmpReg)]; +build_stk([], _TmpReg, nil) -> + [{move,nil,{x,1}}]. + build_call(call_fun, Arity, _Func, none, Dst) -> [{call_fun,Arity}|copy({x,0}, Dst)]; build_call(call_fun, Arity, _Func, {return,Dst,N}, Dst) when is_integer(N) -> @@ -1550,15 +1619,15 @@ build_apply(Arity, {return,Val,N}, _Dst) when is_integer(N) -> build_apply(Arity, none, Dst) -> [{apply,Arity}|copy({x,0}, Dst)]. -cg_instr(put_map, [{atom,assoc},SrcMap|Ss], Dst, Set) -> - Live = get_live(Set), - [{put_map_assoc,{f,0},SrcMap,Dst,Live,{list,Ss}}]; cg_instr(bs_get_tail, [Src], Dst, Set) -> Live = get_live(Set), [{bs_get_tail,Src,Dst,Live}]; cg_instr(bs_get_position, [Ctx], Dst, Set) -> Live = get_live(Set), [{bs_get_position,Ctx,Dst,Live}]; +cg_instr(put_map, [{atom,assoc},SrcMap|Ss], Dst, Set) -> + Live = get_live(Set), + [{put_map_assoc,{f,0},SrcMap,Dst,Live,{list,Ss}}]; cg_instr(Op, Args, Dst, _Set) -> cg_instr(Op, Args, Dst). @@ -1728,7 +1797,7 @@ cg_catch(Agg, T0, Context, St0) -> cg_try(Agg, Tag, T0, Context, St0) -> {Moves0,T1} = cg_extract(T0, Agg, St0), - Moves = order_moves(Moves0, {x,3}), + Moves = order_moves(Moves0), [#cg_set{op=kill_try_tag}|T2] = T1, {T,St} = cg_block(T2, Context, St0), {[{try_case,Tag}|Moves++T],St}. @@ -1780,7 +1849,7 @@ linearize(Blocks) -> Linear = beam_ssa:linearize(Blocks), linearize_1(Linear, Blocks). -linearize_1([{?BADARG_BLOCK,_}|Ls], Blocks) -> +linearize_1([{?EXCEPTION_BLOCK,_}|Ls], Blocks) -> linearize_1(Ls, Blocks); linearize_1([{L,Block0}|Ls], Blocks) -> Block = translate_block(L, Block0, Blocks), @@ -1884,8 +1953,7 @@ setup_args([]) -> []; setup_args([_|_]=Args) -> Moves = gen_moves(Args, 0, []), - Scratch = {x,1+last(sort([length(Args)-1|[X || {x,X} <- Args]]))}, - order_moves(Moves, Scratch). + order_moves(Moves). %% kill_yregs(Anno, #cg{}) -> [{kill,{y,Y}}]. %% Kill Y registers that will not be used again. @@ -1905,47 +1973,48 @@ gen_moves([A|As], I, Acc) -> gen_moves([], _, Acc) -> keysort(3, Acc). -%% order_moves([Move], ScratchReg) -> [Move] +%% order_moves([Move]) -> [Move] %% Orders move instruction so that source registers are not %% destroyed before they are used. If there are cycles %% (such as {move,{x,0},{x,1}}, {move,{x,1},{x,1}}), -%% the scratch register is used to break up the cycle. -%% If possible, the first move of the input list is placed +%% swap instructions will be used to break up the cycle. +%% +%% If possible, the first move of the input list is placed %% last in the result list (to make the move to {x,0} occur %% just before the call to allow the Beam loader to coalesce %% the instructions). -order_moves(Ms, Scr) -> order_moves(Ms, Scr, []). +order_moves(Ms) -> order_moves(Ms, []). -order_moves([{move,_,_}=M|Ms0], ScrReg, Acc0) -> - {Chain,Ms} = collect_chain(Ms0, [M], ScrReg), +order_moves([{move,_,_}=M|Ms0], Acc0) -> + {Chain,Ms} = collect_chain(Ms0, [M]), Acc = reverse(Chain, Acc0), - order_moves(Ms, ScrReg, Acc); -order_moves([], _, Acc) -> Acc. + order_moves(Ms, Acc); +order_moves([], Acc) -> Acc. -collect_chain(Ms, Path, ScrReg) -> - collect_chain(Ms, Path, [], ScrReg). +collect_chain(Ms, Path) -> + collect_chain(Ms, Path, []). -collect_chain([{move,Src,Same}=M|Ms0], [{move,Same,_}|_]=Path, Others, ScrReg) -> +collect_chain([{move,Src,Same}=M|Ms0], [{move,Same,_}|_]=Path, Others) -> case keymember(Src, 3, Path) of false -> - collect_chain(reverse(Others, Ms0), [M|Path], [], ScrReg); + collect_chain(reverse(Others, Ms0), [M|Path], []); true -> - %% There is a cycle, which we must break up. - {break_up_cycle(M, Path, ScrReg),reverse(Others, Ms0)} + %% There is a cycle. + {break_up_cycle(M, Path),reverse(Others, Ms0)} end; -collect_chain([M|Ms], Path, Others, ScrReg) -> - collect_chain(Ms, Path, [M|Others], ScrReg); -collect_chain([], Path, Others, _) -> +collect_chain([M|Ms], Path, Others) -> + collect_chain(Ms, Path, [M|Others]); +collect_chain([], Path, Others) -> {Path,Others}. -break_up_cycle({move,Src,_}=M, Path, ScrReg) -> - [{move,ScrReg,Src},M|break_up_cycle1(Src, Path, ScrReg)]. +break_up_cycle({move,Src,_Dst}=M, Path) -> + break_up_cycle_1(Src, [M|Path], []). -break_up_cycle1(Dst, [{move,Src,Dst}|Path], ScrReg) -> - [{move,Src,ScrReg}|Path]; -break_up_cycle1(Dst, [M|Path], LastMove) -> - [M|break_up_cycle1(Dst, Path, LastMove)]. +break_up_cycle_1(Dst, [{move,_Src,Dst}|Path], Acc) -> + reverse(Acc, Path); +break_up_cycle_1(Dst, [{move,S,D}|Path], Acc) -> + break_up_cycle_1(Dst, Path, [{swap,S,D}|Acc]). %%% %%% General utility functions. diff --git a/lib/compiler/src/beam_ssa_dead.erl b/lib/compiler/src/beam_ssa_dead.erl index e78e4647a8..55ded77d43 100644 --- a/lib/compiler/src/beam_ssa_dead.erl +++ b/lib/compiler/src/beam_ssa_dead.erl @@ -760,8 +760,8 @@ will_succeed_1('=/=', A, '=:=', B) when A =:= B -> no; will_succeed_1('<', A, '=:=', B) when B >= A -> no; will_succeed_1('<', A, '=/=', B) when B >= A -> yes; will_succeed_1('<', A, '<', B) when B >= A -> yes; -will_succeed_1('<', A, '=<', B) when B > A -> yes; -will_succeed_1('<', A, '>=', B) when B > A -> no; +will_succeed_1('<', A, '=<', B) when B >= A -> yes; +will_succeed_1('<', A, '>=', B) when B >= A -> no; will_succeed_1('<', A, '>', B) when B >= A -> no; will_succeed_1('=<', A, '=:=', B) when B > A -> no; @@ -781,9 +781,9 @@ will_succeed_1('>=', A, '>', B) when B < A -> yes; will_succeed_1('>', A, '=:=', B) when B =< A -> no; will_succeed_1('>', A, '=/=', B) when B =< A -> yes; will_succeed_1('>', A, '<', B) when B =< A -> no; -will_succeed_1('>', A, '=<', B) when B < A -> no; +will_succeed_1('>', A, '=<', B) when B =< A -> no; will_succeed_1('>', A, '>=', B) when B =< A -> yes; -will_succeed_1('>', A, '>', B) when B < A -> yes; +will_succeed_1('>', A, '>', B) when B =< A -> yes; will_succeed_1('==', A, '==', B) -> if diff --git a/lib/compiler/src/beam_ssa_lint.erl b/lib/compiler/src/beam_ssa_lint.erl index a003607dab..224095d4c4 100644 --- a/lib/compiler/src/beam_ssa_lint.erl +++ b/lib/compiler/src/beam_ssa_lint.erl @@ -65,13 +65,19 @@ format_error({{_M,F,A},{phi_inside_block, Name, Id}}) -> [F, A, format_var(Name), Id]); format_error({{_M,F,A},{undefined_label_in_phi, Label, I}}) -> io_lib:format("~p/~p: Unknown block label ~p in phi node ~ts", - [F, A, Label, format_instr(I)]). + [F, A, Label, format_instr(I)]); +format_error({{_M,F,A},{succeeded_not_preceded, I}}) -> + io_lib:format("~p/~p: ~ts does not reference the preceding instruction", + [F, A, format_instr(I)]); +format_error({{_M,F,A},{succeeded_not_last, I}}) -> + io_lib:format("~p/~p: ~ts is not the last instruction in its block", + [F, A, format_instr(I)]). format_instr(I) -> [$',beam_ssa_pp:format_instr(I),$']. format_var(V) -> - beam_ssa_pp:format_var(#b_var{name=V}). + beam_ssa_pp:format_var(V). validate_function(F) -> try @@ -86,34 +92,36 @@ validate_function(F) -> erlang:raise(Class, Error, Stack) end. --type defined_vars() :: gb_sets:set(beam_ssa:var_name()). +-type defined_vars() :: gb_sets:set(beam_ssa:argument()). -record(vvars, {blocks :: #{ beam_ssa:label() => beam_ssa:b_blk() }, branch_def_vars :: #{ - %% Describes the variable state at the time of this exact branch (phi - %% node validation). - {From :: beam_ssa:label(), To :: beam_ssa:label()} => defined_vars(), - %% Describes the variable state common to all branches leading to this - %% label (un/redefined variable validation). - beam_ssa:label() => defined_vars() }, + %% Describes the variable state at the time of + %% this exact branch (phi node validation). + {From :: beam_ssa:label(), + To :: beam_ssa:label()} => defined_vars(), + %% Describes the variable state common to all + %% branches leading to this label (un/redefined + %% variable validation). + beam_ssa:label() => defined_vars() }, defined_vars :: defined_vars()}). -spec validate_variables(beam_ssa:b_function()) -> ok. validate_variables(#b_function{ args = Args, bs = Blocks }) -> %% Prefill the mapping with function arguments. - ArgNames = vvars_get_varnames(Args), - DefVars = gb_sets:from_list(ArgNames), + Args = vvars_get_variables(Args), + DefVars = gb_sets:from_list(Args), Entry = 0, State = #vvars{blocks = Blocks, branch_def_vars = #{ Entry => DefVars }, defined_vars = DefVars}, - ok = vvars_assert_unique(Blocks, ArgNames), + ok = vvars_assert_unique(Blocks, Args), vvars_phi_nodes(vvars_block(Entry, State)). %% Checks the uniqueness of all variables across all blocks. --spec vvars_assert_unique(Blocks, [beam_ssa:var_name()]) -> ok when +-spec vvars_assert_unique(Blocks, [beam_ssa:b_var()]) -> ok when Blocks :: #{ beam_ssa:label() => beam_ssa:b_blk() }. vvars_assert_unique(Blocks, Args) -> BlockIs = [Is || #b_blk{is=Is} <- maps:values(Blocks)], @@ -124,12 +132,12 @@ vvars_assert_unique(Blocks, Args) -> ok. -spec vvars_assert_unique_1(Is, Defined) -> ok when - Is :: list(beam_ssa:b_set()), - Defined :: #{ beam_ssa:var_name() => beam_ssa:b_set() }. -vvars_assert_unique_1([#b_set{dst=#b_var{name=DstName}}=I|Is], Defined) -> + Is :: list(beam_ssa:b_set()), + Defined :: #{ beam_ssa:b_var() => beam_ssa:b_set() }. +vvars_assert_unique_1([#b_set{dst=Dst}=I|Is], Defined) -> case Defined of - #{DstName:=Old} -> throw({redefined_variable, DstName, Old, I}); - _ -> vvars_assert_unique_1(Is, Defined#{DstName=>I}) + #{Dst:=Old} -> throw({redefined_variable, Dst, Old, I}); + _ -> vvars_assert_unique_1(Is, Defined#{Dst=>I}) end; vvars_assert_unique_1([], Defined) -> Defined. @@ -141,17 +149,17 @@ vvars_phi_nodes(#vvars{ blocks = Blocks }=State) -> ok. -spec vvars_phi_nodes_1(Is, Id, State) -> ok when - Is :: list(beam_ssa:b_set()), - Id :: beam_ssa:label(), - State :: #vvars{}. + Is :: list(beam_ssa:b_set()), + Id :: beam_ssa:label(), + State :: #vvars{}. vvars_phi_nodes_1([#b_set{ op = phi, args = Phis }=I | Is], Id, State) -> ok = vvars_assert_phi_paths(Phis, I, Id, State), ok = vvars_assert_phi_vars(Phis, I, Id, State), vvars_phi_nodes_1(Is, Id, State); vvars_phi_nodes_1([_ | Is], Id, _State) -> - case [Dst || #b_set{op=phi,dst=#b_var{name=Dst}} <- Is] of - [Name|_] -> - throw({phi_inside_block, Name, Id}); + case [Dst || #b_set{op=phi,dst=Dst} <- Is] of + [Var|_] -> + throw({phi_inside_block, Var, Id}); [] -> ok end; @@ -161,10 +169,10 @@ vvars_phi_nodes_1([], _Id, _State) -> %% Checks whether all paths leading to this phi node are represented, and that %% it doesn't reference any non-existent paths. -spec vvars_assert_phi_paths(Phis, I, Id, State) -> ok when - Phis :: list({beam_ssa:argument(), beam_ssa:label()}), - Id :: beam_ssa:label(), - I :: beam_ssa:b_set(), - State :: #vvars{}. + Phis :: list({beam_ssa:argument(), beam_ssa:label()}), + Id :: beam_ssa:label(), + I :: beam_ssa:b_set(), + State :: #vvars{}. vvars_assert_phi_paths(Phis, I, Id, State) -> BranchKeys = maps:keys(State#vvars.branch_def_vars), RequiredPaths = ordsets:from_list([From || {From, To} <- BranchKeys, To =:= Id]), @@ -173,34 +181,34 @@ vvars_assert_phi_paths(Phis, I, Id, State) -> [_|_]=MissingPaths -> throw({missing_phi_paths, MissingPaths, I}); [] -> ok end. - %% %% The following test is sometimes useful to find missing optimizations. - %% %% It is commented out, though, because it can be triggered by - %% %% by weird but legal code. - %% case ordsets:subtract(ProvidedPaths, RequiredPaths) of - %% [_|_]=GarbagePaths -> throw({garbage_phi_paths, GarbagePaths, I}); - %% [] -> ok - %% end. +%% %% The following test is sometimes useful to find missing optimizations. +%% %% It is commented out, though, because it can be triggered by +%% %% by weird but legal code. +%% case ordsets:subtract(ProvidedPaths, RequiredPaths) of +%% [_|_]=GarbagePaths -> throw({garbage_phi_paths, GarbagePaths, I}); +%% [] -> ok +%% end. %% Checks whether all variables used in this phi node are defined in the branch %% they arrived on. -spec vvars_assert_phi_vars(Phis, I, Id, State) -> ok when - Phis :: list({beam_ssa:argument(), beam_ssa:label()}), - Id :: beam_ssa:label(), - I :: beam_ssa:b_set(), - State :: #vvars{}. + Phis :: list({beam_ssa:argument(), beam_ssa:label()}), + Id :: beam_ssa:label(), + I :: beam_ssa:b_set(), + State :: #vvars{}. vvars_assert_phi_vars(Phis, I, Id, #vvars{blocks=Blocks, branch_def_vars=BranchDefVars}) -> Vars = [{Var, From} || {#b_var{}=Var, From} <- Phis], - foreach(fun({#b_var{name=VarName}, From}) -> + foreach(fun({Var, From}) -> BranchKey = {From, Id}, case BranchDefVars of #{BranchKey:=DefVars} -> - case gb_sets:is_member(VarName, DefVars) of + case gb_sets:is_member(Var, DefVars) of true -> ok; - false -> throw({unknown_variable, VarName, I}) + false -> throw({unknown_variable, Var, I}) end; #{} -> - throw({unknown_phi_variable, VarName, BranchKey, I}) + throw({unknown_phi_variable, Var, BranchKey, I}) end end, Vars), Labels = [From || {#b_literal{},From} <- Phis], @@ -214,32 +222,44 @@ vvars_assert_phi_vars(Phis, I, Id, #vvars{blocks=Blocks, end, Labels). -spec vvars_block(Id, State) -> #vvars{} when - Id :: beam_ssa:label(), - State :: #vvars{}. + Id :: beam_ssa:label(), + State :: #vvars{}. vvars_block(Id, State0) -> #{ Id := #b_blk{ is = Is, last = Terminator} } = State0#vvars.blocks, #{ Id := DefVars } = State0#vvars.branch_def_vars, State = State0#vvars{ defined_vars = DefVars }, vvars_terminator(Terminator, Id, vvars_block_1(Is, State)). --spec vvars_block_1(Blocks, State) -> #vvars{} when - Blocks :: list(beam_ssa:b_blk()), - State :: #vvars{}. +-spec vvars_block_1(Is, State) -> #vvars{} when + Is :: list(#b_set{}), + State :: #vvars{}. vvars_block_1([], State) -> State; -vvars_block_1([#b_set{ dst = #b_var{ name = DstName }, op = phi } | Is], State0) -> +vvars_block_1([#b_set{dst=OpVar,args=OpArgs}=I, + #b_set{op=succeeded,args=[OpVar],dst=SuccVar}], State) -> + ok = vvars_assert_args(OpArgs, I, State), + vvars_save_var(SuccVar, vvars_save_var(OpVar, State)); +vvars_block_1([#b_set{op=succeeded,args=Args}=I | [_|_]], State) -> + ok = vvars_assert_args(Args, I, State), + %% 'succeeded' must be the last instruction in its block. + throw({succeeded_not_last, I}); +vvars_block_1([#b_set{op=succeeded,args=Args}=I], State)-> + ok = vvars_assert_args(Args, I, State), + %% 'succeeded' must be be directly preceded by the operation it checks. + throw({succeeded_not_preceded, I}); +vvars_block_1([#b_set{ dst = Dst, op = phi } | Is], State) -> %% We don't check phi node arguments at this point since we may not have %% visited their definition yet. They'll be handled later on in %% vvars_phi_nodes/1 after all blocks are processed. - vvars_block_1(Is, vvars_save_var(DstName, State0)); -vvars_block_1([#b_set{ dst = #b_var{ name = DstName }, args = Args }=I | Is], State0) -> - ok = vvars_assert_args(Args, I, State0), - vvars_block_1(Is, vvars_save_var(DstName, State0)). + vvars_block_1(Is, vvars_save_var(Dst, State)); +vvars_block_1([#b_set{ dst = Dst, args = Args }=I | Is], State) -> + ok = vvars_assert_args(Args, I, State), + vvars_block_1(Is, vvars_save_var(Dst, State)). -spec vvars_terminator(Terminator, From, State) -> #vvars{} when - Terminator :: beam_ssa:terminator(), - From :: beam_ssa:label(), - State :: #vvars{}. + Terminator :: beam_ssa:terminator(), + From :: beam_ssa:label(), + State :: #vvars{}. vvars_terminator(#b_ret{ arg = Arg }=I, _From, State) -> ok = vvars_assert_args([Arg], I, State), State; @@ -264,62 +284,62 @@ vvars_terminator(#b_br{ bool = Arg, succ = Succ, fail = Fail }=I, From, State) - vvars_terminator_1(Labels, From, State). -spec vvars_terminator_1(Labels, From, State) -> #vvars{} when - Labels :: list(beam_ssa:label()), - From :: beam_ssa:label(), - State :: #vvars{}. + Labels :: list(beam_ssa:label()), + From :: beam_ssa:label(), + State :: #vvars{}. vvars_terminator_1(Labels0, From, State0) -> %% Filter out all branches that have already been taken. This should result %% in either all of Labels0 or an empty list. Labels = [To || To <- Labels0, - not maps:is_key({From, To}, State0#vvars.branch_def_vars)], + not maps:is_key({From, To}, State0#vvars.branch_def_vars)], true = Labels =:= Labels0 orelse Labels =:= [], %Assertion State1 = foldl(fun(To, State) -> - vvars_save_branch(From, To, State) + vvars_save_branch(From, To, State) end, State0, Labels), foldl(fun(To, State) -> - vvars_block(To, State) + vvars_block(To, State) end, State1, Labels). %% Gets all variable names in args, ignoring literals etc --spec vvars_get_varnames(Args) -> list(beam_ssa:var_name()) when - Args :: list(beam_ssa:argument()). -vvars_get_varnames(Args) -> - [Name || #b_var{ name = Name } <- Args]. +-spec vvars_get_variables(Args) -> list(beam_ssa:b_var()) when + Args :: list(beam_ssa:argument()). +vvars_get_variables(Args) -> + [Var || #b_var{}=Var <- Args]. %% Checks that all variables in Args are defined in all paths leading to the %% current State. -spec vvars_assert_args(Args, I, State) -> ok when - Args :: list(beam_ssa:argument()), - I :: beam_ssa:terminator() | beam_ssa:b_set(), - State :: #vvars{}. + Args :: list(beam_ssa:argument()), + I :: beam_ssa:terminator() | beam_ssa:b_set(), + State :: #vvars{}. vvars_assert_args(Args, I, #vvars{defined_vars=DefVars}=State) -> foreach(fun(#b_remote{mod=Mod,name=Name}) -> vvars_assert_args([Mod,Name], I, State); - (#b_var{name=Name}) -> - case gb_sets:is_member(Name, DefVars) of + (#b_var{}=Var) -> + case gb_sets:is_member(Var, DefVars) of true -> ok; - false -> throw({unknown_variable,Name,I}) + false -> throw({unknown_variable,Var,I}) end; (_) -> ok end, Args). %% Checks that all given labels are defined in State. -spec vvars_assert_labels(Labels, I, State) -> ok when - Labels :: list(beam_ssa:label()), - I :: beam_ssa:terminator(), - State :: #vvars{}. + Labels :: list(beam_ssa:label()), + I :: beam_ssa:terminator(), + State :: #vvars{}. vvars_assert_labels(Labels, I, #vvars{blocks=Blocks}) -> foreach(fun(Label) -> - case maps:is_key(Label, Blocks) of - false -> throw({unknown_block, Label, I}); - true -> ok - end + case maps:is_key(Label, Blocks) of + false -> throw({unknown_block, Label, I}); + true -> ok + end end, Labels). -spec vvars_save_branch(From, To, State) -> #vvars{} when - From :: beam_ssa:label(), - To :: beam_ssa:label(), - State :: #vvars{}. + From :: beam_ssa:label(), + To :: beam_ssa:label(), + State :: #vvars{}. vvars_save_branch(From, To, State) -> DefVars = State#vvars.defined_vars, Branches0 = State#vvars.branch_def_vars, @@ -335,15 +355,15 @@ vvars_save_branch(From, To, State) -> end. -spec vvars_merge_branches(New, Existing) -> defined_vars() when - New :: defined_vars(), - Existing :: defined_vars(). + New :: defined_vars(), + Existing :: defined_vars(). vvars_merge_branches(New, Existing) -> gb_sets:intersection(New, Existing). --spec vvars_save_var(VarName, State) -> #vvars{} when - VarName :: beam_ssa:var_name(), - State :: #vvars{}. -vvars_save_var(VarName, State0) -> +-spec vvars_save_var(Var, State) -> #vvars{} when + Var :: #b_var{}, + State :: #vvars{}. +vvars_save_var(Var, State0) -> %% vvars_assert_unique guarantees that variables are never set twice. - DefVars = gb_sets:insert(VarName, State0#vvars.defined_vars), + DefVars = gb_sets:insert(Var, State0#vvars.defined_vars), State0#vvars{ defined_vars = DefVars }. diff --git a/lib/compiler/src/beam_ssa_opt.erl b/lib/compiler/src/beam_ssa_opt.erl index d87c66c272..580abf4ed9 100644 --- a/lib/compiler/src/beam_ssa_opt.erl +++ b/lib/compiler/src/beam_ssa_opt.erl @@ -145,7 +145,8 @@ prologue_passes(Opts) -> ?PASS(ssa_opt_linearize), ?PASS(ssa_opt_tuple_size), ?PASS(ssa_opt_record), - ?PASS(ssa_opt_cse), %Helps the first type pass. + ?PASS(ssa_opt_cse), % Helps the first type pass. + ?PASS(ssa_opt_live), % ... ?PASS(ssa_opt_type_start)], passes_1(Ps, Opts). @@ -157,6 +158,9 @@ repeated_passes(Opts) -> ?PASS(ssa_opt_dead), ?PASS(ssa_opt_cse), ?PASS(ssa_opt_tail_phis), + ?PASS(ssa_opt_sink), + ?PASS(ssa_opt_tuple_size), + ?PASS(ssa_opt_record), ?PASS(ssa_opt_type_continue)], %Must run after ssa_opt_dead to %clean up phi nodes. passes_1(Ps, Opts). @@ -172,8 +176,8 @@ epilogue_passes(Opts) -> ?PASS(ssa_opt_bsm), ?PASS(ssa_opt_bsm_units), ?PASS(ssa_opt_bsm_shortcut), - ?PASS(ssa_opt_blockify), ?PASS(ssa_opt_sink), + ?PASS(ssa_opt_blockify), ?PASS(ssa_opt_merge_blocks), ?PASS(ssa_opt_get_tuple_element), ?PASS(ssa_opt_trim_unreachable)], @@ -898,6 +902,7 @@ cse_suitable(#b_set{}) -> false. -record(fs, {s=undefined :: 'undefined' | 'cleared', regs=#{} :: #{beam_ssa:b_var():=beam_ssa:b_var()}, + vars=cerl_sets:new() :: cerl_sets:set(), fail=none :: 'none' | beam_ssa:label(), non_guards :: gb_sets:set(beam_ssa:label()), bs :: beam_ssa:block_map() @@ -910,22 +915,39 @@ ssa_opt_float({#st{ssa=Linear0,cnt=Count0}=St, FuncDb}) -> {Linear,Count} = float_opt(Linear0, Count0, Fs), {St#st{ssa=Linear,cnt=Count}, FuncDb}. -float_blk_is_in_guard(#b_blk{last=#b_br{fail=F}}, #fs{non_guards=NonGuards}) -> - not gb_sets:is_member(F, NonGuards); -float_blk_is_in_guard(#b_blk{}, #fs{}) -> +%% The fconv instruction doesn't support jumping to a fail label, so we have to +%% skip this optimization if the fail block is a guard. +%% +%% We also skip the optimization in blocks that always fail, as it's both +%% difficult and pointless to rewrite them to use float ops. +float_can_optimize_blk(#b_blk{last=#b_br{bool=#b_var{},fail=F}}, + #fs{non_guards=NonGuards}) -> + gb_sets:is_member(F, NonGuards); +float_can_optimize_blk(#b_blk{}, #fs{}) -> false. +float_opt([{L,#b_blk{is=[#b_set{op=exception_trampoline,args=[Var]}]}=Blk0} | + Bs0], Count0, Fs) -> + %% If we've replaced a BIF with float operations, we'll have a lot of extra + %% blocks that jump to the same failure block, which may have a trampoline + %% that refers to the original operation. + %% + %% Since the point of the trampoline is to keep the BIF from being removed + %% by liveness optimization, we can discard it as the liveness pass leaves + %% floats alone. + Blk = case cerl_sets:is_element(Var, Fs#fs.vars) of + true -> Blk0#b_blk{is=[]}; + false -> Blk0 + end, + {Bs, Count} = float_opt(Bs0, Count0, Fs), + {[{L,Blk}|Bs],Count}; float_opt([{L,Blk}|Bs0], Count0, Fs) -> - case float_blk_is_in_guard(Blk, Fs) of + case float_can_optimize_blk(Blk, Fs) of true -> - %% This block is inside a guard. Don't do - %% any floating point optimizations. - {Bs,Count} = float_opt(Bs0, Count0, Fs), - {[{L,Blk}|Bs],Count}; + float_opt_1(L, Blk, Bs0, Count0, Fs); false -> - %% This block is not inside a guard. - %% We can do the optimization. - float_opt_1(L, Blk, Bs0, Count0, Fs) + {Bs,Count} = float_opt(Bs0, Count0, Fs), + {[{L,Blk}|Bs],Count} end; float_opt([], Count, _Fs) -> {[],Count}. @@ -1004,12 +1026,12 @@ float_maybe_flush(Blk0, #fs{s=cleared,fail=Fail,bs=Blocks}=Fs0, Count0) -> #b_blk{last=#b_br{bool=#b_var{},succ=Succ}=Br} = Blk0, %% If the success block starts with a floating point operation, we can - %% defer flushing to that block as long as it isn't a guard. + %% defer flushing to that block as long as it's suitable for optimization. #b_blk{is=Is} = SuccBlk = map_get(Succ, Blocks), - SuccIsGuard = float_blk_is_in_guard(SuccBlk, Fs0), + CanOptimizeSucc = float_can_optimize_blk(SuccBlk, Fs0), case Is of - [#b_set{anno=#{float_op:=_}}|_] when not SuccIsGuard -> + [#b_set{anno=#{float_op:=_}}|_] when CanOptimizeSucc -> %% No flush needed. {[],Blk0,Fs0,Count0}; _ -> @@ -1065,21 +1087,22 @@ float_opt_is([], Fs, _Count, _Acc) -> none. float_make_op(#b_set{op={bif,Op},dst=Dst,args=As0}=I0, - Ts, #fs{s=S,regs=Rs0}=Fs, Count0) -> + Ts, #fs{s=S,regs=Rs0,vars=Vs0}=Fs, Count0) -> {As1,Rs1,Count1} = float_load(As0, Ts, Rs0, Count0, []), {As,Is0} = unzip(As1), {Fr,Count2} = new_reg('@fr', Count1), FrDst = #b_var{name=Fr}, I = I0#b_set{op={float,Op},dst=FrDst,args=As}, + Vs = cerl_sets:add_element(Dst, Vs0), Rs = Rs1#{Dst=>FrDst}, Is = append(Is0) ++ [I], case S of undefined -> {Ignore,Count} = new_reg('@ssa_ignore', Count2), C = #b_set{op={float,clearerror},dst=#b_var{name=Ignore}}, - {[C|Is],Fs#fs{s=cleared,regs=Rs},Count}; + {[C|Is],Fs#fs{s=cleared,regs=Rs,vars=Vs},Count}; cleared -> - {Is,Fs#fs{regs=Rs},Count2} + {Is,Fs#fs{regs=Rs,vars=Vs},Count2} end. float_load([A|As], [T|Ts], Rs0, Count0, Acc) -> @@ -1208,34 +1231,31 @@ live_opt_is([#b_set{op=phi,dst=Dst}=I|Is], Live, Acc) -> false -> live_opt_is(Is, Live, Acc) end; -live_opt_is([#b_set{op=succeeded,dst=SuccDst=SuccDstVar, - args=[Dst]}=SuccI, - #b_set{dst=Dst}=I|Is], Live0, Acc) -> - case gb_sets:is_member(Dst, Live0) of - true -> - Live1 = gb_sets:add(Dst, Live0), - Live = gb_sets:delete_any(SuccDst, Live1), - live_opt_is([I|Is], Live, [SuccI|Acc]); - false -> - case live_opt_unused(I) of - {replace,NewI0} -> - NewI = NewI0#b_set{dst=SuccDstVar}, - live_opt_is([NewI|Is], Live0, Acc); - keep -> - case gb_sets:is_member(SuccDst, Live0) of - true -> - Live1 = gb_sets:add(Dst, Live0), - Live = gb_sets:delete(SuccDst, Live1), - live_opt_is([I|Is], Live, [SuccI|Acc]); - false -> - live_opt_is([I|Is], Live0, Acc) - end - end +live_opt_is([#b_set{op=succeeded,dst=SuccDst,args=[MapDst]}=SuccI, + #b_set{op=get_map_element,dst=MapDst}=MapI | Is], + Live0, Acc) -> + case {gb_sets:is_member(SuccDst, Live0), + gb_sets:is_member(MapDst, Live0)} of + {true, true} -> + Live = gb_sets:delete(SuccDst, Live0), + live_opt_is([MapI | Is], Live, [SuccI | Acc]); + {true, false} -> + %% 'get_map_element' is unused; replace 'succeeded' with + %% 'has_map_field' + NewI = MapI#b_set{op=has_map_field,dst=SuccDst}, + live_opt_is([NewI | Is], Live0, Acc); + {false, true} -> + %% 'succeeded' is unused (we know it will succeed); discard it and + %% keep 'get_map_element' + live_opt_is([MapI | Is], Live0, Acc); + {false, false} -> + live_opt_is(Is, Live0, Acc) end; live_opt_is([#b_set{dst=Dst}=I|Is], Live0, Acc) -> case gb_sets:is_member(Dst, Live0) of true -> - Live1 = gb_sets:union(Live0, gb_sets:from_ordset(beam_ssa:used(I))), + LiveUsed = gb_sets:from_ordset(beam_ssa:used(I)), + Live1 = gb_sets:union(Live0, LiveUsed), Live = gb_sets:delete(Dst, Live1), live_opt_is(Is, Live, [I|Acc]); false -> @@ -1243,17 +1263,14 @@ live_opt_is([#b_set{dst=Dst}=I|Is], Live0, Acc) -> true -> live_opt_is(Is, Live0, Acc); false -> - Live = gb_sets:union(Live0, gb_sets:from_ordset(beam_ssa:used(I))), + LiveUsed = gb_sets:from_ordset(beam_ssa:used(I)), + Live = gb_sets:union(Live0, LiveUsed), live_opt_is(Is, Live, [I|Acc]) end end; live_opt_is([], Live, Acc) -> {Acc,Live}. -live_opt_unused(#b_set{op=get_map_element}=Set) -> - {replace,Set#b_set{op=has_map_field}}; -live_opt_unused(_) -> keep. - %%% %%% Optimize binary matching. %%% @@ -1939,6 +1956,10 @@ verify_merge_is(_) -> is_merge_allowed(_, #b_blk{}, #b_blk{is=[#b_set{op=peek_message}|_]}) -> false; +is_merge_allowed(_, #b_blk{}, #b_blk{is=[#b_set{op=exception_trampoline}|_]}) -> + false; +is_merge_allowed(_, #b_blk{is=[#b_set{op=exception_trampoline}|_]}, #b_blk{}) -> + false; is_merge_allowed(L, #b_blk{last=#b_br{}}=Blk, #b_blk{is=Is}) -> %% The predecessor block must have exactly one successor (L) for %% the merge to be safe. @@ -1977,9 +1998,7 @@ is_merge_allowed(_, #b_blk{last=#b_switch{}}, #b_blk{}) -> %%% extracted values. %%% -ssa_opt_sink({#st{ssa=Blocks0}=St, FuncDb}) -> - Linear = beam_ssa:linearize(Blocks0), - +ssa_opt_sink({#st{ssa=Linear}=St, FuncDb}) -> %% Create a map with all variables that define get_tuple_element %% instructions. The variable name map to the block it is defined in. case def_blocks(Linear) of @@ -1988,10 +2007,12 @@ ssa_opt_sink({#st{ssa=Blocks0}=St, FuncDb}) -> {St, FuncDb}; [_|_]=Defs0 -> Defs = maps:from_list(Defs0), - {do_ssa_opt_sink(Linear, Defs, St), FuncDb} + {do_ssa_opt_sink(Defs, St), FuncDb} end. -do_ssa_opt_sink(Linear, Defs, #st{ssa=Blocks0}=St) -> +do_ssa_opt_sink(Defs, #st{ssa=Linear}=St) -> + Blocks0 = maps:from_list(Linear), + %% Now find all the blocks that use variables defined by get_tuple_element %% instructions. Used = used_blocks(Linear, Defs, []), @@ -2016,7 +2037,8 @@ do_ssa_opt_sink(Linear, Defs, #st{ssa=Blocks0}=St) -> From = map_get(V, Defs), move_defs(V, From, To, A) end, Blocks0, DefLoc), - St#st{ssa=Blocks}. + + St#st{ssa=beam_ssa:linearize(Blocks)}. def_blocks([{L,#b_blk{is=Is}}|Bs]) -> def_blocks_is(Is, L, def_blocks(Bs)); @@ -2049,6 +2071,7 @@ unsuitable_1([{L,#b_blk{is=[#b_set{op=Op}|_]}}|Bs]) -> Unsuitable = case Op of bs_extract -> true; bs_put -> true; + exception_trampoline -> true; {float,_} -> true; landingpad -> true; peek_message -> true; @@ -2257,13 +2280,15 @@ non_guards(Linear) -> non_guards_1([{L,#b_blk{is=Is}}|Bs]) -> case Is of + [#b_set{op=exception_trampoline}|_] -> + [L | non_guards_1(Bs)]; [#b_set{op=landingpad}|_] -> [L | non_guards_1(Bs)]; _ -> non_guards_1(Bs) end; non_guards_1([]) -> - [?BADARG_BLOCK]. + [?EXCEPTION_BLOCK]. rel2fam(S0) -> S1 = sofs:relation(S0), diff --git a/lib/compiler/src/beam_ssa_pre_codegen.erl b/lib/compiler/src/beam_ssa_pre_codegen.erl index 7f816b9802..bf2503a2a5 100644 --- a/lib/compiler/src/beam_ssa_pre_codegen.erl +++ b/lib/compiler/src/beam_ssa_pre_codegen.erl @@ -108,7 +108,8 @@ functions([], _Ps, _UseBSM3) -> []. intervals=[] :: [{b_var(),[range()]}], res=[] :: [{b_var(),reservation()}] | #{b_var():=reservation()}, regs=#{} :: #{b_var():=ssa_register()}, - extra_annos=[] :: [{atom(),term()}] + extra_annos=[] :: [{atom(),term()}], + location :: term() }). -define(PASS(N), {N,fun N/1}). @@ -119,7 +120,9 @@ passes(Opts) -> %% Preliminaries. ?PASS(fix_bs), + ?PASS(exception_trampolines), ?PASS(sanitize), + ?PASS(match_fail_instructions), case FixTuples of false -> ignore; true -> ?PASS(fix_tuples) @@ -164,7 +167,9 @@ passes(Opts) -> function(#b_function{anno=Anno,args=Args,bs=Blocks0,cnt=Count0}=F0, Ps, UseBSM3) -> try - St0 = #st{ssa=Blocks0,args=Args,use_bsm3=UseBSM3,cnt=Count0}, + Location = maps:get(location, Anno, none), + St0 = #st{ssa=Blocks0,args=Args,use_bsm3=UseBSM3, + cnt=Count0,location=Location}, St = compile:run_sub_passes(Ps, St0), #st{ssa=Blocks,cnt=Count,regs=Regs,extra_annos=ExtraAnnos} = St, F1 = add_extra_annos(F0, ExtraAnnos), @@ -598,6 +603,10 @@ bs_instrs([{L,#b_blk{is=Is0}=Blk}|Bs], CtxChain, Acc0) -> bs_instrs([], _, Acc) -> reverse(Acc). +bs_instrs_is([#b_set{op=succeeded}=I|Is], CtxChain, Acc) -> + %% This instruction refers to a specific operation, so we must not + %% substitute the context argument. + bs_instrs_is(Is, CtxChain, [I | Acc]); bs_instrs_is([#b_set{op=Op,args=Args0}=I0|Is], CtxChain, Acc) -> Args = [bs_subst_ctx(A, CtxChain) || A <- Args0], I1 = I0#b_set{args=Args}, @@ -691,6 +700,54 @@ legacy_bs_is([I|Is], Last, IsYreg, Count, Copies, Acc) -> legacy_bs_is([], _Last, _IsYreg, Count, Copies, Acc) -> {reverse(Acc),Count,Copies}. +%% exception_trampolines(St0) -> St. +%% +%% Removes the "exception trampolines" that were added to prevent exceptions +%% from being optimized away. + +exception_trampolines(#st{ssa=Blocks0}=St) -> + RPO = reverse(beam_ssa:rpo(Blocks0)), + Blocks = et_1(RPO, #{}, #{}, Blocks0), + St#st{ssa=Blocks}. + +et_1([L | Ls], Trampolines, Exceptions, Blocks) -> + #{ L := #b_blk{is=Is,last=Last0}=Block0 } = Blocks, + case {Is, Last0} of + {[#b_set{op=exception_trampoline,args=[Arg]}], #b_br{succ=Succ}} -> + et_1(Ls, + Trampolines#{ L => Succ }, + Exceptions#{ L => Arg }, + maps:remove(L, Blocks)); + {_, #b_br{succ=Same,fail=Same}} when Same =:= ?EXCEPTION_BLOCK -> + %% The exception block is just a marker saying that we should raise + %% an exception (= {f,0}) instead of jumping to a particular fail + %% block. Since it's not a reachable block we can't allow + %% unconditional jumps to it except through a trampoline. + error({illegal_jump_to_exception_block, L}); + {_, #b_br{succ=Same,fail=Same}} + when map_get(Same, Trampolines) =:= ?EXCEPTION_BLOCK -> + %% This block always fails at runtime (and we are not in a + %% try/catch); rewrite the terminator to a return. + Last = #b_ret{arg=map_get(Same, Exceptions)}, + Block = Block0#b_blk{last=Last}, + et_1(Ls, Trampolines, Exceptions, Blocks#{ L := Block }); + {_, #b_br{succ=Succ0,fail=Fail0}} -> + Succ = maps:get(Succ0, Trampolines, Succ0), + Fail = maps:get(Fail0, Trampolines, Fail0), + if + Succ =/= Succ0; Fail =/= Fail0 -> + Last = Last0#b_br{succ=Succ,fail=Fail}, + Block = Block0#b_blk{last=Last}, + et_1(Ls, Trampolines, Exceptions, Blocks#{ L := Block }); + Succ =:= Succ0, Fail =:= Fail0 -> + et_1(Ls, Trampolines, Exceptions, Blocks) + end; + {_, _} -> + et_1(Ls, Trampolines, Exceptions, Blocks) + end; +et_1([], _Trampolines, _Exceptions, Blocks) -> + Blocks. + %% sanitize(St0) -> St. %% Remove constructs that can cause problems later: %% @@ -856,6 +913,114 @@ prune_phi(#b_set{args=Args0}=Phi, Reachable) -> gb_sets:is_element(Pred, Reachable)], Phi#b_set{args=Args}. +%%% Rewrite certain calls to erlang:error/{1,2} to specialized +%%% instructions: +%%% +%%% erlang:error({badmatch,Value}) => badmatch Value +%%% erlang:error({case_clause,Value}) => case_end Value +%%% erlang:error({try_clause,Value}) => try_case_end Value +%%% erlang:error(if_clause) => if_end +%%% erlang:error(function_clause, Args) => jump FuncInfoLabel +%%% +%%% In SSA code, we represent those instructions as a 'match_fail' +%%% instruction with the name of the BEAM instruction as the first +%%% argument. + +match_fail_instructions(#st{ssa=Blocks0,args=Args,location=Location}=St) -> + Ls = maps:to_list(Blocks0), + Info = {length(Args),Location}, + Blocks = match_fail_instrs_1(Ls, Info, Blocks0), + St#st{ssa=Blocks}. + +match_fail_instrs_1([{L,#b_blk{is=Is0}=Blk}|Bs], Arity, Blocks0) -> + case match_fail_instrs_blk(Is0, Arity, []) of + none -> + match_fail_instrs_1(Bs, Arity, Blocks0); + Is -> + Blocks = Blocks0#{L:=Blk#b_blk{is=Is}}, + match_fail_instrs_1(Bs, Arity, Blocks) + end; +match_fail_instrs_1([], _Arity, Blocks) -> Blocks. + +match_fail_instrs_blk([#b_set{op=put_tuple,dst=Dst, + args=[#b_literal{val=Tag},Val]}, + #b_set{op=call, + args=[#b_remote{mod=#b_literal{val=erlang}, + name=#b_literal{val=error}}, + Dst]}=Call|Is], + _Arity, Acc) -> + match_fail_instr(Call, Tag, Val, Is, Acc); +match_fail_instrs_blk([#b_set{op=call, + args=[#b_remote{mod=#b_literal{val=erlang}, + name=#b_literal{val=error}}, + #b_literal{val={Tag,Val}}]}=Call|Is], + _Arity, Acc) -> + match_fail_instr(Call, Tag, #b_literal{val=Val}, Is, Acc); +match_fail_instrs_blk([#b_set{op=call, + args=[#b_remote{mod=#b_literal{val=erlang}, + name=#b_literal{val=error}}, + #b_literal{val=if_clause}]}=Call|Is], + _Arity, Acc) -> + I = Call#b_set{op=match_fail,args=[#b_literal{val=if_end}]}, + reverse(Acc, [I|Is]); +match_fail_instrs_blk([#b_set{op=call,anno=Anno, + args=[#b_remote{mod=#b_literal{val=erlang}, + name=#b_literal{val=error}}, + #b_literal{val=function_clause}, + Stk]}=Call], + {Arity,Location}, Acc) -> + case match_fail_stk(Stk, Acc, [], []) of + {[_|_]=Vars,Is} when length(Vars) =:= Arity -> + case maps:get(location, Anno, none) of + Location -> + I = Call#b_set{op=match_fail, + args=[#b_literal{val=function_clause}|Vars]}, + Is ++ [I]; + _ -> + %% erlang:error/2 has a different location than the + %% func_info instruction at the beginning of the function + %% (probably because of inlining). Keep the original call. + reverse(Acc, [Call]) + end; + _ -> + %% Either the stacktrace could not be picked apart (for example, + %% if the call to erlang:error/2 was handwritten) or the number + %% of arguments in the stacktrace was different from the arity + %% of the host function (because it is the implementation of a + %% fun). Keep the original call. + reverse(Acc, [Call]) + end; +match_fail_instrs_blk([I|Is], Arity, Acc) -> + match_fail_instrs_blk(Is, Arity, [I|Acc]); +match_fail_instrs_blk(_, _, _) -> + none. + +match_fail_instr(Call, Tag, Val, Is, Acc) -> + Op = case Tag of + badmatch -> Tag; + case_clause -> case_end; + try_clause -> try_case_end; + _ -> none + end, + case Op of + none -> + none; + _ -> + I = Call#b_set{op=match_fail,args=[#b_literal{val=Op},Val]}, + reverse(Acc, [I|Is]) + end. + +match_fail_stk(#b_var{}=V, [#b_set{op=put_list,dst=V,args=[H,T]}|Is], IAcc, VAcc) -> + match_fail_stk(T, Is, IAcc, [H|VAcc]); +match_fail_stk(#b_literal{val=[H|T]}, Is, IAcc, VAcc) -> + match_fail_stk(#b_literal{val=T}, Is, IAcc, [#b_literal{val=H}|VAcc]); +match_fail_stk(#b_literal{val=[]}, [], IAcc, VAcc) -> + {reverse(VAcc),IAcc}; +match_fail_stk(T, [#b_set{op=Op}=I|Is], IAcc, VAcc) + when Op =:= bs_get_tail; Op =:= bs_set_position -> + match_fail_stk(T, Is, [I|IAcc], VAcc); +match_fail_stk(_, _, _, _) -> none. + %%% %%% Fix tuples. %%% @@ -911,9 +1076,8 @@ use_set_tuple_element(#st{ssa=Blocks0}=St) -> Blocks = use_ste_1(RPO, Uses, Blocks0), St#st{ssa=Blocks}. -use_ste_1([L|Ls], Uses, Blocks0) -> - {Blk0,Blocks} = use_ste_across(L, Uses, Blocks0), - #b_blk{is=Is0} = Blk0, +use_ste_1([L|Ls], Uses, Blocks) -> + #b_blk{is=Is0} = Blk0 = map_get(L, Blocks), case use_ste_is(Is0, Uses) of Is0 -> use_ste_1(Ls, Uses, Blocks); @@ -976,69 +1140,6 @@ extract_ste(#b_set{op=call,dst=Dst, end; extract_ste(#b_set{}) -> none. -%%% Optimize accross blocks within a try/catch block. - -use_ste_across(L, Uses, Blocks) -> - case map_get(L, Blocks) of - #b_blk{last=#b_br{bool=#b_var{}}}=Blk -> - try - use_ste_across_1(L, Blk, Uses, Blocks) - catch - throw:not_possible -> - {Blk,Blocks} - end; - #b_blk{}=Blk -> - {Blk,Blocks} - end. - -use_ste_across_1(L, Blk0, Uses, Blocks0) -> - #b_blk{is=IsThis,last=#b_br{bool=Bool,succ=Next}} = Blk0, - case reverse(IsThis) of - [#b_set{op=succeeded,dst=Bool,args=[Result]}=Succ0, - #b_set{op=call,args=[#b_remote{}|_],dst=Result}=Call1|Prefix] -> - case is_single_use(Bool, Uses) andalso - is_n_uses(2, Result, Uses) of - true -> ok; - false -> throw(not_possible) - end, - Call2 = use_ste_across_next(Next, Uses, Blocks0), - Is = [Call1,Call2], - case use_ste_is(Is, decrement_uses(Result, Uses)) of - [#b_set{}=Call,#b_set{op=set_tuple_element}=Ste] -> - Blocks1 = use_ste_fix_next(Ste, Next, Blocks0), - Succ = Succ0#b_set{args=[Call#b_set.dst]}, - Blk = Blk0#b_blk{is=reverse(Prefix, [Call,Succ])}, - Blocks = Blocks1#{L:=Blk}, - {Blk,Blocks}; - _ -> - throw(not_possible) - end; - _ -> - throw(not_possible) - end. - -use_ste_across_next(Next, Uses, Blocks) -> - case map_get(Next, Blocks) of - #b_blk{is=[#b_set{op=call,dst=Result,args=[#b_remote{}|_]}=Call, - #b_set{op=succeeded,dst=Bool,args=[Result]}], - last=#b_br{bool=Bool}} -> - case is_single_use(Bool, Uses) andalso - is_n_uses(2, Result, Uses) of - true -> ok; - false -> throw(not_possible) - end, - Call; - #b_blk{} -> - throw(not_possible) - end. - -use_ste_fix_next(Ste, Next, Blocks) -> - Blk0 = map_get(Next, Blocks), - #b_blk{is=[#b_set{op=call},#b_set{op=succeeded}],last=Br0} = Blk0, - Br = beam_ssa:normalize(Br0#b_br{bool=#b_literal{val=true}}), - Blk = Blk0#b_blk{is=[Ste],last=Br}, - Blocks#{Next:=Blk}. - %% Count how many times each variable is used. count_uses(Blocks) -> @@ -1048,7 +1149,7 @@ count_uses_blk([#b_blk{is=Is,last=Last}|Bs], CountMap0) -> F = fun(I, CountMap) -> foldl(fun(Var, Acc) -> case Acc of - #{Var:=3} -> Acc; + #{Var:=2} -> Acc; #{Var:=C} -> Acc#{Var:=C+1}; #{} -> Acc#{Var=>1} end @@ -1058,16 +1159,6 @@ count_uses_blk([#b_blk{is=Is,last=Last}|Bs], CountMap0) -> count_uses_blk(Bs, CountMap); count_uses_blk([], CountMap) -> CountMap. -decrement_uses(V, Uses) -> - #{V:=C} = Uses, - Uses#{V:=C-1}. - -is_n_uses(N, V, Uses) -> - case Uses of - #{V:=N} -> true; - #{} -> false - end. - is_single_use(V, Uses) -> case Uses of #{V:=1} -> true; @@ -1189,10 +1280,10 @@ place_frame_here(L, Blocks, Doms, Frames) -> Descendants = beam_ssa:rpo([L], Blocks), PhiPredecessors = phi_predecessors(L, Blocks), MustDominate = ordsets:from_list(PhiPredecessors ++ Descendants), - Dominates = all(fun(?BADARG_BLOCK) -> + Dominates = all(fun(?EXCEPTION_BLOCK) -> %% This block defines no variables and calls %% erlang:error(badarg). It does not matter - %% whether L dominates ?BADARG_BLOCK or not; + %% whether L dominates ?EXCEPTION_BLOCK or not; %% it is still safe to put the frame in L. true; (Bl) -> @@ -1250,14 +1341,9 @@ need_frame_1([#b_set{op=call,args=[Func|_]}|Is], Context) -> #b_remote{mod=#b_literal{val=Mod}, name=#b_literal{val=Name}, arity=Arity} when is_atom(Mod), is_atom(Name) -> - case erl_bifs:is_exit_bif(Mod, Name, Arity) of - true -> - false; - false -> - Context =:= body orelse - Is =/= [] orelse - is_trap_bif(Mod, Name, Arity) - end; + Context =:= body orelse + Is =/= [] orelse + is_trap_bif(Mod, Name, Arity); #b_remote{} -> %% This is an apply(), which always needs a frame. true; @@ -1340,9 +1426,9 @@ recv_common(_Defs, none, _Blocks) -> %% in the tail position of a function. []; recv_common(Defs, Exit, Blocks) -> - {ExitDefs,ExitUsed} = beam_ssa:def_used([Exit], Blocks), + {ExitDefs,ExitUnused} = beam_ssa:def_unused([Exit], Defs, Blocks), Def = ordsets:subtract(Defs, ExitDefs), - ordsets:intersection(Def, ExitUsed). + ordsets:subtract(Def, ExitUnused). %% recv_crit_edges([RemoveMessageLabel], LoopExit, %% Blocks0, Count0) -> {Blocks,Count}. @@ -1447,9 +1533,9 @@ exit_predecessors([], _Exit, _Blocks) -> []. %% later used within a clause of the receive. fix_receive([L|Ls], Defs, Blocks0, Count0) -> - {RmDefs,Used0} = beam_ssa:def_used([L], Blocks0), + {RmDefs,Unused} = beam_ssa:def_unused([L], Defs, Blocks0), Def = ordsets:subtract(Defs, RmDefs), - Used = ordsets:intersection(Def, Used0), + Used = ordsets:subtract(Def, Unused), {NewVars,Count} = new_vars([Base || #b_var{name=Base} <- Used], Count0), Ren = zip(Used, NewVars), Blocks1 = beam_ssa:rename_vars(Ren, [L], Blocks0), @@ -1483,8 +1569,8 @@ find_loop_exit(_, _) -> %% loop exit block. none. -find_loop_exit_1([?BADARG_BLOCK|Ls], RmSet, Dominators) -> - %% ?BADARG_BLOCK is a marker and not an actual block, so it is not +find_loop_exit_1([?EXCEPTION_BLOCK|Ls], RmSet, Dominators) -> + %% ?EXCEPTION_BLOCK is a marker and not an actual block, so it is not %% the block we are looking for. find_loop_exit_1(Ls, RmSet, Dominators); find_loop_exit_1([L|Ls], RmSet, Dominators) -> @@ -1756,7 +1842,7 @@ collect_yregs([], Yregs) -> Yregs. copy_retval_2([L|Ls], Yregs, Copy0, Blocks0, Count0) -> #b_blk{is=Is0,last=Last} = Blk = map_get(L, Blocks0), RC = case {Last,Ls} of - {#b_br{succ=Succ,fail=?BADARG_BLOCK},[Succ|_]} -> + {#b_br{succ=Succ,fail=?EXCEPTION_BLOCK},[Succ|_]} -> true; {_,_} -> false @@ -2105,8 +2191,8 @@ reserve_yregs(#st{frames=Frames}=St0) -> reserve_yregs_1(L, #st{ssa=Blocks0,cnt=Count0,res=Res0}=St) -> Blk = map_get(L, Blocks0), Yregs = beam_ssa:get_anno(yregs, Blk), - {Def,Used} = beam_ssa:def_used([L], Blocks0), - UsedYregs = ordsets:intersection(Yregs, Used), + {Def,Unused} = beam_ssa:def_unused([L], Yregs, Blocks0), + UsedYregs = ordsets:subtract(Yregs, Unused), DefBefore = ordsets:subtract(UsedYregs, Def), {BeforeVars,Blocks,Count} = rename_vars(DefBefore, L, Blocks0, Count0), InsideVars = ordsets:subtract(UsedYregs, DefBefore), @@ -2495,9 +2581,9 @@ reserve_xregs_is([], Res, Xs, _Used) -> {Res,Xs}. %% Pick up register hints from the successors of this blocks. -reserve_terminator(_L, _Is, #b_br{bool=#b_var{},succ=Succ,fail=?BADARG_BLOCK}, +reserve_terminator(_L, _Is, #b_br{bool=#b_var{},succ=Succ,fail=?EXCEPTION_BLOCK}, _Blocks, XsMap, _Res) -> - %% We know that no variables are used at ?BADARG_BLOCK, so + %% We know that no variables are used at ?EXCEPTION_BLOCK, so %% any register hints from the success blocks are safe to use. map_get(Succ, XsMap); reserve_terminator(L, Is, #b_br{bool=#b_var{},succ=Succ,fail=Fail}, diff --git a/lib/compiler/src/beam_ssa_share.erl b/lib/compiler/src/beam_ssa_share.erl index 73983bd34a..fa728992f8 100644 --- a/lib/compiler/src/beam_ssa_share.erl +++ b/lib/compiler/src/beam_ssa_share.erl @@ -117,8 +117,8 @@ share_terminator(_Last, _Blocks) -> none. %% possible if the blocks are not equivalent, as that is the common %% case. -are_equivalent(_Succ, _, ?BADARG_BLOCK, _, _Blocks) -> - %% ?BADARG_BLOCK is special. Sharing could be incorrect. +are_equivalent(_Succ, _, ?EXCEPTION_BLOCK, _, _Blocks) -> + %% ?EXCEPTION_BLOCK is special. Sharing could be incorrect. false; are_equivalent(_Succ, #b_blk{is=Is1,last=#b_ret{arg=RetVal1}=Ret1}, _Fail, #b_blk{is=Is2,last=#b_ret{arg=RetVal2}=Ret2}, _Blocks) -> diff --git a/lib/compiler/src/beam_ssa_type.erl b/lib/compiler/src/beam_ssa_type.erl index 3c06c83e2e..d93191c689 100644 --- a/lib/compiler/src/beam_ssa_type.erl +++ b/lib/compiler/src/beam_ssa_type.erl @@ -22,11 +22,12 @@ -export([opt_start/4, opt_continue/4, opt_finish/3]). -include("beam_ssa_opt.hrl"). --import(lists, [all/2,any/2,droplast/1,foldl/3,last/1,member/2, - keyfind/3,reverse/1,reverse/2, - sort/1,split/2]). +-include("beam_types.hrl"). --define(UNICODE_INT, #t_integer{elements={0,16#10FFFF}}). +-import(lists, [all/2,any/2,droplast/1,duplicate/2,foldl/3,last/1,member/2, + keyfind/3,reverse/1,sort/1,split/2,zip/2]). + +-define(UNICODE_MAX, (16#10FFFF)). -record(d, {ds :: #{beam_ssa:b_var():=beam_ssa:b_set()}, @@ -37,21 +38,6 @@ sub = #{} :: #{beam_ssa:b_var():=beam_ssa:value()}, ret_type = [] :: [type()]}). --define(ATOM_SET_SIZE, 5). - -%% Records that represent type information. --record(t_atom, {elements=any :: 'any' | [atom()]}). --record(t_integer, {elements=any :: 'any' | {integer(),integer()}}). --record(t_bs_match, {type :: type()}). --record(t_tuple, {size=0 :: integer(), - exact=false :: boolean(), - %% Known element types (1-based index), unknown elements are - %% are assumed to be 'any'. - elements=#{} :: #{ non_neg_integer() => type() }}). - --type type() :: 'any' | 'none' | - #t_atom{} | #t_integer{} | #t_bs_match{} | #t_tuple{} | - {'binary',pos_integer()} | 'cons' | 'float' | 'list' | 'map' | 'nil' | 'number'. -type type_db() :: #{beam_ssa:var_name():=type()}. -spec opt_start(Linear, Args, Anno, FuncDb) -> {Linear, FuncDb} when @@ -98,7 +84,8 @@ join_arg_types(Args, ArgTypes, Anno) -> end, Ts0, ParamTypes). join_arg_types_1([Arg | Args], [TM | TMs], Ts) when map_size(TM) =/= 0 -> - join_arg_types_1(Args, TMs, Ts#{ Arg => join(maps:values(TM))}); + Type = beam_types:join(maps:values(TM)), + join_arg_types_1(Args, TMs, Ts#{ Arg => Type }); join_arg_types_1([Arg | Args], [_TM | TMs], Ts) -> join_arg_types_1(Args, TMs, Ts#{ Arg => any }); join_arg_types_1([], [], Ts) -> @@ -122,7 +109,7 @@ opt_continue_1(Linear0, Args, Id, Ts, FuncDb0) -> D = #d{ func_db=FuncDb0, func_id=Id, ds=Defs, - ls=#{0=>Ts,?BADARG_BLOCK=>#{}}, + ls=#{0=>Ts,?EXCEPTION_BLOCK=>#{}}, once=UsedOnce }, {Linear, FuncDb, NewRet} = opt(Linear0, D, []), @@ -157,43 +144,15 @@ opt_finish_1([Arg | Args], [TypeMap | TypeMaps], ParamInfo) map_size(TypeMap) =:= 0 -> opt_finish_1(Args, TypeMaps, ParamInfo); opt_finish_1([Arg | Args], [TypeMap | TypeMaps], ParamInfo0) -> - case join(maps:values(TypeMap)) of - any -> - opt_finish_1(Args, TypeMaps, ParamInfo0); - none -> - %% This function will never be called. Pretend that we don't - %% know the type for this argument. - opt_finish_1(Args, TypeMaps, ParamInfo0); - JoinedType -> - JoinedType = verified_type(JoinedType), - ParamInfo = ParamInfo0#{ Arg => validator_anno(JoinedType) }, - opt_finish_1(Args, TypeMaps, ParamInfo) - end; + JoinedType = beam_types:join(maps:values(TypeMap)), + ParamInfo = case JoinedType of + any -> ParamInfo0; + _ -> ParamInfo0#{ Arg => JoinedType } + end, + opt_finish_1(Args, TypeMaps, ParamInfo); opt_finish_1([], [], ParamInfo) -> ParamInfo. -validator_anno(#t_tuple{size=Size,exact=Exact,elements=Elements0}) -> - Elements = maps:fold(fun(Index, Type, Acc) -> - Key = beam_validator:type_anno(integer, Index), - Acc#{ Key => validator_anno(Type) } - end, #{}, Elements0), - beam_validator:type_anno(tuple, Size, Exact, Elements); -validator_anno(#t_integer{elements={Same,Same}}) -> - beam_validator:type_anno(integer, Same); -validator_anno(#t_integer{}) -> - beam_validator:type_anno(integer); -validator_anno(float) -> - beam_validator:type_anno(float); -validator_anno(#t_atom{elements=[Val]}) -> - beam_validator:type_anno(atom, Val); -validator_anno(#t_atom{}=A) -> - case t_is_boolean(A) of - true -> beam_validator:type_anno(bool); - false -> beam_validator:type_anno(atom) - end; -validator_anno(T) -> - beam_validator:type_anno(T). - get_func_id(Anno) -> #{func_info:={_Mod, Name, Arity}} = Anno, #b_local{name=#b_literal{val=Name}, arity=Arity}. @@ -212,27 +171,17 @@ opt([], D, Acc) -> opt_1(L, #b_blk{is=Is0,last=Last0}=Blk0, Bs, Ts0, #d{ds=Ds0,sub=Sub0,func_db=Fdb0}=D0, Acc) -> - case opt_is(Is0, Ts0, Ds0, Fdb0, D0, Sub0, []) of - {Is,Ts,Ds,Fdb,Sub} -> - D1 = D0#d{ds=Ds,sub=Sub,func_db=Fdb}, - Last1 = simplify_terminator(Last0, Sub, Ts, Ds), - Last = opt_terminator(Last1, Ts, Ds), - D = update_successors(Last, Ts, D1), - Blk = Blk0#b_blk{is=Is,last=Last}, - opt(Bs, D, [{L,Blk}|Acc]); - {no_return,Ret,Is,Ds,Fdb,Sub} -> - %% This call will never reach the successor block. - %% Rewrite the terminator to a 'ret', and remove - %% all type information for this label. That can - %% potentially narrow the type of the phi node - %% in the former successor. - Ls = maps:remove(L, D0#d.ls), - RetType = join([none|D0#d.ret_type]), - D = D0#d{ds=Ds,ls=Ls,sub=Sub, - func_db=Fdb,ret_type=[RetType]}, - Blk = Blk0#b_blk{is=Is,last=Ret}, - opt(Bs, D, [{L,Blk}|Acc]) - end. + {Is,Ts,Ds,Fdb,Sub} = opt_is(Is0, Ts0, Ds0, Fdb0, D0, Sub0, []), + + D1 = D0#d{ds=Ds,sub=Sub,func_db=Fdb}, + + Last1 = simplify_terminator(Last0, Sub, Ts, Ds), + Last2 = opt_terminator(Last1, Ts, Ds), + + {Last, D} = update_successors(Last2, Ts, D1), + + Blk = Blk0#b_blk{is=Is,last=Last}, + opt(Bs, D, [{L,Blk}|Acc]). simplify_terminator(#b_br{bool=Bool}=Br, Sub, Ts, _Ds) -> Br#b_br{bool=simplify_arg(Bool, Sub, Ts)}; @@ -265,203 +214,138 @@ opt_is([#b_set{op=phi,dst=Dst,args=Args0}=I0|Is], Ds = Ds0#{Dst=>I}, opt_is(Is, Ts, Ds, Fdb, D, Sub0, [I|Acc]) end; -opt_is([#b_set{op=call,args=Args0,dst=Dst}=I0|Is], +opt_is([#b_set{op=call,args=Args0}=I0|Is], + Ts, Ds, Fdb0, D, Sub, Acc) -> + Args = simplify_args(Args0, Sub, Ts), + I1 = beam_ssa:normalize(I0#b_set{args=Args}), + {I, Fdb} = opt_call(I1, Ts, Fdb0, D), + opt_simplify(I, Is, Ts, Ds, Fdb, D, Sub, Acc); +opt_is([#b_set{op=make_fun,args=Args0}=I0|Is], Ts0, Ds0, Fdb0, D, Sub0, Acc) -> Args = simplify_args(Args0, Sub0, Ts0), I1 = beam_ssa:normalize(I0#b_set{args=Args}), - {Ts1,Ds,Fdb,I2} = opt_call(I1, D, Ts0, Ds0, Fdb0), - case {map_get(Dst, Ts1),Is} of - {Type,[#b_set{op=succeeded}]} when Type =/= none -> - %% This call instruction is inside a try/catch - %% block. Don't attempt to simplify it. - opt_is(Is, Ts1, Ds, Fdb, D, Sub0, [I2|Acc]); - {none,[#b_set{op=succeeded}]} -> - %% This call instruction is inside a try/catch - %% block, but we know it will never return and - %% later optimizations may try to exploit that. - %% - %% For example, if we have an expression that - %% either returns this call or a tuple, we know - %% that the expression always returns a tuple - %% and can turn a later element/3 into - %% get_tuple_element. - %% - %% This is sound but difficult to validate in a - %% meaningful way as try/catch currently forces - %% us to maintain the illusion that the success - %% block is reachable even when its not, so we - %% disable the optimization to keep things - %% simple. - Ts = Ts1#{ Dst := any }, - opt_is(Is, Ts, Ds, Fdb, D, Sub0, [I2|Acc]); - {none,_} -> - %% This call never returns. The rest of the - %% instructions will not be executed. - Ret = #b_ret{arg=Dst}, - {no_return,Ret,reverse(Acc, [I2]),Ds,Fdb,Sub0}; - {_,_} -> - case simplify_call(I2) of - #b_set{}=I -> - opt_is(Is, Ts1, Ds, Fdb, D, Sub0, [I|Acc]); - #b_literal{}=Lit -> - Sub = Sub0#{Dst=>Lit}, - Ts = maps:remove(Dst, Ts1), - opt_is(Is, Ts, Ds0, Fdb, D, Sub, Acc); - #b_var{}=Var -> - Ts = maps:remove(Dst, Ts1), - Sub = Sub0#{Dst=>Var}, - opt_is(Is, Ts, Ds0, Fdb, D, Sub, Acc) - end - end; -opt_is([#b_set{op=succeeded,args=[Arg],dst=Dst}=I], + {Ts,Ds,Fdb,I} = opt_make_fun(I1, D, Ts0, Ds0, Fdb0), + opt_is(Is, Ts, Ds, Fdb, D, Sub0, [I|Acc]); +opt_is([#b_set{op=succeeded,args=[Arg],dst=Dst,anno=Anno}=I], Ts0, Ds0, Fdb, D, Sub0, Acc) -> - case Ds0 of - #{ Arg := #b_set{op=call} } -> - %% The success check of a call is part of exception handling and - %% must not be optimized away. We still have to update its type - %% though. - Ts = update_types(I, Ts0, Ds0), - Ds = Ds0#{Dst=>I}, - - opt_is([], Ts, Ds, Fdb, D, Sub0, [I|Acc]); - #{} -> - Args = simplify_args([Arg], Sub0, Ts0), - Type = type(succeeded, Args, Ts0, Ds0), - case get_literal_from_type(Type) of - #b_literal{}=Lit -> - Sub = Sub0#{Dst=>Lit}, - opt_is([], Ts0, Ds0, Fdb, D, Sub, Acc); - none -> - Ts = Ts0#{Dst=>Type}, - Ds = Ds0#{Dst=>I}, - opt_is([], Ts, Ds, Fdb, D, Sub0, [I|Acc]) - end + Type = case Ds0 of + #{ Arg := #b_set{op=call} } -> + %% Calls can always throw exceptions and their return types + %% are what they return on success, so we must avoid + %% simplifying arguments in case `Arg` would become a + %% literal, which would trick 'succeeded' into thinking it + %% can't fail. + type(succeeded, [Arg], Anno, Ts0, Ds0); + #{} -> + Args = simplify_args([Arg], Sub0, Ts0), + type(succeeded, Args, Anno, Ts0, Ds0) + end, + case beam_types:get_singleton_value(Type) of + {ok, Lit} -> + Sub = Sub0#{ Dst => #b_literal{val=Lit} }, + opt_is([], Ts0, Ds0, Fdb, D, Sub, Acc); + error -> + Ts = Ts0#{ Dst => Type }, + Ds = Ds0#{ Dst => I }, + opt_is([], Ts, Ds, Fdb, D, Sub0, [I | Acc]) end; -opt_is([#b_set{args=Args0,dst=Dst}=I0|Is], - Ts0, Ds0, Fdb, D, Sub0, Acc) -> - Args = simplify_args(Args0, Sub0, Ts0), - I1 = beam_ssa:normalize(I0#b_set{args=Args}), - case simplify(I1, Ts0) of +opt_is([#b_set{args=Args0}=I0|Is], + Ts, Ds, Fdb, D, Sub, Acc) -> + Args = simplify_args(Args0, Sub, Ts), + I = beam_ssa:normalize(I0#b_set{args=Args}), + opt_simplify(I, Is, Ts, Ds, Fdb, D, Sub, Acc); +opt_is([], Ts, Ds, Fdb, _D, Sub, Acc) -> + {reverse(Acc), Ts, Ds, Fdb, Sub}. + +opt_simplify(#b_set{dst=Dst}=I0, Is, Ts0, Ds0, Fdb, D, Sub0, Acc) -> + case simplify(I0, Ts0) of #b_set{}=I2 -> I = beam_ssa:normalize(I2), Ts = update_types(I, Ts0, Ds0), - Ds = Ds0#{Dst=>I}, + Ds = Ds0#{ Dst => I }, opt_is(Is, Ts, Ds, Fdb, D, Sub0, [I|Acc]); #b_literal{}=Lit -> - Sub = Sub0#{Dst=>Lit}, + Sub = Sub0#{ Dst => Lit }, opt_is(Is, Ts0, Ds0, Fdb, D, Sub, Acc); #b_var{}=Var -> case Is of [#b_set{op=succeeded,dst=SuccDst,args=[Dst]}] -> - %% We must remove this 'succeeded' instruction. - Sub = Sub0#{Dst=>Var,SuccDst=>#b_literal{val=true}}, + %% We must remove this 'succeeded' instruction since the + %% variable it checks is gone. + Sub = Sub0#{ Dst => Var, SuccDst => #b_literal{val=true} }, opt_is([], Ts0, Ds0, Fdb, D, Sub, Acc); _ -> - Sub = Sub0#{Dst=>Var}, + Sub = Sub0#{ Dst => Var}, opt_is(Is, Ts0, Ds0, Fdb, D, Sub, Acc) end - end; -opt_is([], Ts, Ds, Fdb, _D, Sub, Acc) -> - {reverse(Acc), Ts, Ds, Fdb, Sub}. - -simplify_call(#b_set{op=call,args=[#b_remote{}=Rem|Args]}=I) -> - case Rem of - #b_remote{mod=#b_literal{val=Mod}, - name=#b_literal{val=Name}} -> - case erl_bifs:is_pure(Mod, Name, length(Args)) of - true -> - simplify_remote_call(Mod, Name, Args, I); - false -> - I - end; - #b_remote{} -> - I - end; -simplify_call(I) -> I. - -%% Simplify a remote call to a pure BIF. -simplify_remote_call(erlang, '++', [#b_literal{val=[]},Tl], _I) -> - Tl; -simplify_remote_call(erlang, setelement, - [#b_literal{val=Pos}, - #b_literal{val=Tuple}, - #b_var{}=Value], I) - when is_integer(Pos), 1 =< Pos, Pos =< tuple_size(Tuple) -> - %% Position is a literal integer and the shape of the - %% tuple is known. - Els0 = [#b_literal{val=El} || El <- tuple_to_list(Tuple)], - {Bef,[_|Aft]} = split(Pos - 1, Els0), - Els = Bef ++ [Value|Aft], - I#b_set{op=put_tuple,args=Els}; -simplify_remote_call(Mod, Name, Args0, I) -> - case make_literal_list(Args0) of - none -> - I; - Args -> - %% The arguments are literals. Try to evaluate the BIF. - try apply(Mod, Name, Args) of - Val -> - case cerl:is_literal_term(Val) of - true -> - #b_literal{val=Val}; - false -> - %% The value can't be expressed as a literal - %% (e.g. a pid). - I - end - catch - _:_ -> - %% Failed. Don't bother trying to optimize - %% the call. - I - end end. -opt_call(#b_set{dst=Dst,args=[#b_local{}=Callee|Args]}=I0, D, Ts0, Ds0, Fdb0) -> - {Ts, Ds, I} = opt_local_call(I0, Ts0, Ds0, Fdb0), +opt_call(#b_set{dst=Dst,args=[#b_local{}=Callee|Args]}=I0, Ts, Fdb0, D) -> + I = opt_local_call_return(I0, Callee, Fdb0), case Fdb0 of #{ Callee := #func_info{exported=false,arg_types=ArgTypes0}=Info } -> + %% Match contexts are treated as bitstrings when optimizing + %% arguments, as we don't yet support removing the + %% "bs_start_match3" instruction. + Types = [case raw_type(Arg, Ts) of + #t_bs_context{} -> #t_bitstring{}; + Type -> Type + end || Arg <- Args], + %% Update the argument types of *this exact call*, the types %% will be joined later when the callee is optimized. CallId = {D#d.func_id, Dst}, - ArgTypes = update_arg_types(Args, ArgTypes0, CallId, Ts0), + ArgTypes = update_arg_types(Types, ArgTypes0, CallId), Fdb = Fdb0#{ Callee => Info#func_info{arg_types=ArgTypes} }, - {Ts, Ds, Fdb, I}; + {I, Fdb}; #{} -> %% We can't narrow the argument types of exported functions as they %% can receive anything as part of an external call. - {Ts, Ds, Fdb0, I} + {I, Fdb0} end; -opt_call(#b_set{dst=Dst}=I, _D, Ts0, Ds0, Fdb) -> +opt_call(I, _Ts, Fdb, _D) -> + {I, Fdb}. + +opt_local_call_return(I, Callee, Fdb) -> + case Fdb of + #{ Callee := #func_info{ret_type=[Type]} } when Type =/= any -> + beam_ssa:add_anno(result_type, Type, I); + #{} -> + I + end. + +%% While we have no way to know which arguments a fun will be called with, we +%% do know its free variables and can update their types as if this were a +%% local call. +opt_make_fun(#b_set{op=make_fun, + dst=Dst, + args=[#b_local{}=Callee | FreeVars]}=I, + D, Ts0, Ds0, Fdb0) -> Ts = update_types(I, Ts0, Ds0), Ds = Ds0#{ Dst => I }, - {Ts, Ds, Fdb, I}. + case Fdb0 of + #{ Callee := #func_info{exported=false,arg_types=ArgTypes0}=Info } -> + ArgCount = Callee#b_local.arity - length(FreeVars), -opt_local_call(#b_set{dst=Dst,args=[Id|_]}=I0, Ts0, Ds0, Fdb) -> - Type = case Fdb of - #{ Id := #func_info{ret_type=[T]} } -> T; - #{} -> any - end, - I = case Type of - any -> I0; - none -> I0; - _ -> beam_ssa:add_anno(result_type, validator_anno(Type), I0) - end, - Ts = Ts0#{ Dst => Type }, - Ds = Ds0#{ Dst => I }, - {Ts, Ds, I}. - -update_arg_types([Arg | Args], [TypeMap0 | TypeMaps], CallId, Ts) -> - %% Match contexts are treated as bitstrings when optimizing arguments, as - %% we don't yet support removing the "bs_start_match3" instruction. - NewType = case get_type(Arg, Ts) of - #t_bs_match{} -> {binary, 1}; - Type -> Type - end, - TypeMap = TypeMap0#{ CallId => NewType }, - [TypeMap | update_arg_types(Args, TypeMaps, CallId, Ts)]; -update_arg_types([], [], _CallId, _Ts) -> + FVTypes = [raw_type(FreeVar, Ts) || FreeVar <- FreeVars], + Types = duplicate(ArgCount, any) ++ FVTypes, + + CallId = {D#d.func_id, Dst}, + ArgTypes = update_arg_types(Types, ArgTypes0, CallId), + + Fdb = Fdb0#{ Callee => Info#func_info{arg_types=ArgTypes} }, + {Ts, Ds, Fdb, I}; + #{} -> + %% We can't narrow the argument types of exported functions as they + %% can receive anything as part of an external call. + {Ts, Ds, Fdb0, I} + end. + +update_arg_types([ArgType | ArgTypes], [TypeMap0 | TypeMaps], CallId) -> + TypeMap = TypeMap0#{ CallId => ArgType }, + [TypeMap | update_arg_types(ArgTypes, TypeMaps, CallId)]; +update_arg_types([], [], _CallId) -> []. simplify(#b_set{op={bif,'and'},args=Args}=I, Ts) -> @@ -487,8 +371,10 @@ simplify(#b_set{op={bif,'or'},args=Args}=I, Ts) -> I end; simplify(#b_set{op={bif,element},args=[#b_literal{val=Index},Tuple]}=I0, Ts) -> - case t_tuple_size(get_type(Tuple, Ts)) of - {_,Size} when is_integer(Index), 1 =< Index, Index =< Size -> + case normalized_type(Tuple, Ts) of + #t_tuple{size=Size} when is_integer(Index), + 1 =< Index, + Index =< Size -> I = I0#b_set{op=get_tuple_element, args=[Tuple,#b_literal{val=Index-1}]}, simplify(I, Ts); @@ -496,67 +382,97 @@ simplify(#b_set{op={bif,element},args=[#b_literal{val=Index},Tuple]}=I0, Ts) -> eval_bif(I0, Ts) end; simplify(#b_set{op={bif,hd},args=[List]}=I, Ts) -> - case get_type(List, Ts) of + case normalized_type(List, Ts) of cons -> I#b_set{op=get_hd}; _ -> eval_bif(I, Ts) end; simplify(#b_set{op={bif,tl},args=[List]}=I, Ts) -> - case get_type(List, Ts) of + case normalized_type(List, Ts) of cons -> I#b_set{op=get_tl}; _ -> eval_bif(I, Ts) end; simplify(#b_set{op={bif,size},args=[Term]}=I, Ts) -> - case get_type(Term, Ts) of + case normalized_type(Term, Ts) of #t_tuple{} -> simplify(I#b_set{op={bif,tuple_size}}, Ts); _ -> eval_bif(I, Ts) end; simplify(#b_set{op={bif,tuple_size},args=[Term]}=I, Ts) -> - case get_type(Term, Ts) of + case normalized_type(Term, Ts) of #t_tuple{size=Size,exact=true} -> #b_literal{val=Size}; _ -> I end; -simplify(#b_set{op={bif,'=='},args=Args}=I, Ts) -> - Types = get_types(Args, Ts), - EqEq = case {meet(Types),join(Types)} of - {none,any} -> true; - {#t_integer{},#t_integer{}} -> true; - {float,float} -> true; - {{binary,_},_} -> true; - {#t_atom{},_} -> true; - {_,_} -> false - end, +simplify(#b_set{op={bif,is_function},args=[Fun,#b_literal{val=Arity}]}=I, Ts) + when is_integer(Arity), Arity >= 0 -> + case normalized_type(Fun, Ts) of + #t_fun{arity=any} -> + I; + #t_fun{arity=Arity} -> + #b_literal{val=true}; + any -> + I; + _ -> + #b_literal{val=false} + end; +simplify(#b_set{op={bif,Op0},args=Args}=I, Ts) when Op0 =:= '=='; Op0 =:= '/=' -> + Types = normalized_types(Args, Ts), + EqEq0 = case {beam_types:meet(Types),beam_types:join(Types)} of + {none,any} -> true; + {#t_integer{},#t_integer{}} -> true; + {float,float} -> true; + {#t_bitstring{},_} -> true; + {#t_atom{},_} -> true; + {_,_} -> false + end, + EqEq = EqEq0 orelse any_non_numeric_argument(Args, Ts), case EqEq of true -> - simplify(I#b_set{op={bif,'=:='}}, Ts); + Op = case Op0 of + '==' -> '=:='; + '/=' -> '=/=' + end, + simplify(I#b_set{op={bif,Op}}, Ts); false -> eval_bif(I, Ts) end; simplify(#b_set{op={bif,'=:='},args=[Same,Same]}, _Ts) -> #b_literal{val=true}; -simplify(#b_set{op={bif,'=:='},args=[A1,_A2]=Args}=I, Ts) -> - [T1,T2] = get_types(Args, Ts), - case meet(T1, T2) of +simplify(#b_set{op={bif,'=:='},args=[LHS,RHS]}=I, Ts) -> + LType = raw_type(LHS, Ts), + RType = raw_type(RHS, Ts), + case beam_types:meet(LType, RType) of none -> #b_literal{val=false}; _ -> - case {t_is_boolean(T1),T2} of + case {beam_types:is_boolean_type(LType), + beam_types:normalize(RType)} of {true,#t_atom{elements=[true]}} -> %% Bool =:= true ==> Bool - A1; + LHS; + {true,#t_atom{elements=[false]}} -> + %% Bool =:= false ==> not Bool + %% + %% This will be further optimized to eliminate the + %% 'not', swapping the success and failure + %% branches in the br instruction. If LHS comes + %% from a type test (such as is_atom/1) or a + %% comparison operator (such as >=) that can be + %% translated to test instruction, this + %% optimization will eliminate one instruction. + simplify(I#b_set{op={bif,'not'},args=[LHS]}, Ts); {_,_} -> eval_bif(I, Ts) end end; simplify(#b_set{op={bif,Op},args=Args}=I, Ts) -> - Types = get_types(Args, Ts), + Types = normalized_types(Args, Ts), case is_float_op(Op, Types) of false -> eval_bif(I, Ts); @@ -565,20 +481,15 @@ simplify(#b_set{op={bif,Op},args=Args}=I, Ts) -> eval_bif(beam_ssa:add_anno(float_op, AnnoArgs, I), Ts) end; simplify(#b_set{op=get_tuple_element,args=[Tuple,#b_literal{val=N}]}=I, Ts) -> - case get_type(Tuple, Ts) of - #t_tuple{size=Size,elements=Es} when Size > N -> - ElemType = get_element_type(N + 1, Es), - case get_literal_from_type(ElemType) of - #b_literal{}=Lit -> Lit; - none -> I - end; - none -> - %% Will never be executed because of type conflict. - %% #b_literal{val=ignored}; - I + #t_tuple{size=Size,elements=Es} = normalized_type(Tuple, Ts), + true = Size > N, %Assertion. + ElemType = beam_types:get_element_type(N + 1, Es), + case beam_types:get_singleton_value(ElemType) of + {ok, Val} -> #b_literal{val=Val}; + error -> I end; simplify(#b_set{op=is_nonempty_list,args=[Src]}=I, Ts) -> - case get_type(Src, Ts) of + case normalized_type(Src, Ts) of any -> I; list -> I; cons -> #b_literal{val=true}; @@ -586,7 +497,7 @@ simplify(#b_set{op=is_nonempty_list,args=[Src]}=I, Ts) -> end; simplify(#b_set{op=is_tagged_tuple, args=[Src,#b_literal{val=Size},#b_literal{}=Tag]}=I, Ts) -> - simplify_is_record(I, get_type(Src, Ts), Size, Tag, Ts); + simplify_is_record(I, normalized_type(Src, Ts), Size, Tag, Ts); simplify(#b_set{op=put_list,args=[#b_literal{val=H}, #b_literal{val=T}]}, _Ts) -> #b_literal{val=[H|T]}; @@ -599,8 +510,97 @@ simplify(#b_set{op=wait_timeout,args=[#b_literal{val=0}]}, _Ts) -> #b_literal{val=true}; simplify(#b_set{op=wait_timeout,args=[#b_literal{val=infinity}]}=I, _Ts) -> I#b_set{op=wait,args=[]}; +simplify(#b_set{op=call,args=[#b_remote{}=Rem|Args]}=I, _Ts) -> + case Rem of + #b_remote{mod=#b_literal{val=Mod}, + name=#b_literal{val=Name}} -> + case erl_bifs:is_pure(Mod, Name, length(Args)) of + true -> + simplify_remote_call(Mod, Name, Args, I); + false -> + I + end; + #b_remote{} -> + I + end; simplify(I, _Ts) -> I. +%% Simplify a remote call to a pure BIF. +simplify_remote_call(erlang, '++', [#b_literal{val=[]},Tl], _I) -> + Tl; +simplify_remote_call(erlang, setelement, + [#b_literal{val=Pos}, + #b_literal{val=Tuple}, + #b_var{}=Value], I) + when is_integer(Pos), 1 =< Pos, Pos =< tuple_size(Tuple) -> + %% Position is a literal integer and the shape of the + %% tuple is known. + Els0 = [#b_literal{val=El} || El <- tuple_to_list(Tuple)], + {Bef,[_|Aft]} = split(Pos - 1, Els0), + Els = Bef ++ [Value|Aft], + I#b_set{op=put_tuple,args=Els}; +simplify_remote_call(Mod, Name, Args0, I) -> + case make_literal_list(Args0) of + none -> + I; + Args -> + %% The arguments are literals. Try to evaluate the BIF. + try apply(Mod, Name, Args) of + Val -> + case cerl:is_literal_term(Val) of + true -> + #b_literal{val=Val}; + false -> + %% The value can't be expressed as a literal + %% (e.g. a pid). + I + end + catch + _:_ -> + %% Failed. Don't bother trying to optimize + %% the call. + I + end + end. + +any_non_numeric_argument([#b_literal{val=Lit}|_], _Ts) -> + is_non_numeric(Lit); +any_non_numeric_argument([#b_var{}=V|T], Ts) -> + is_non_numeric_type(raw_type(V, Ts)) orelse any_non_numeric_argument(T, Ts); +any_non_numeric_argument([], _Ts) -> false. + +is_non_numeric([H|T]) -> + is_non_numeric(H) andalso is_non_numeric(T); +is_non_numeric(Tuple) when is_tuple(Tuple) -> + is_non_numeric_tuple(Tuple, tuple_size(Tuple)); +is_non_numeric(Map) when is_map(Map) -> + %% Note that 17.x and 18.x compare keys in different ways. + %% Be very conservative -- require that both keys and values + %% are non-numeric. + is_non_numeric(maps:to_list(Map)); +is_non_numeric(Num) when is_number(Num) -> + false; +is_non_numeric(_) -> true. + +is_non_numeric_tuple(Tuple, El) when El >= 1 -> + is_non_numeric(element(El, Tuple)) andalso + is_non_numeric_tuple(Tuple, El-1); +is_non_numeric_tuple(_Tuple, 0) -> true. + +is_non_numeric_type(#t_atom{}) -> true; +is_non_numeric_type(#t_bitstring{}) -> true; +is_non_numeric_type(nil) -> true; +is_non_numeric_type(#t_tuple{size=Size,exact=true,elements=Types}) + when map_size(Types) =:= Size -> + is_non_numeric_tuple_type(Size, Types); +is_non_numeric_type(_) -> false. + +is_non_numeric_tuple_type(0, _Types) -> + true; +is_non_numeric_tuple_type(Pos, Types) -> + is_non_numeric_type(map_get(Pos, Types)) andalso + is_non_numeric_tuple_type(Pos - 1, Types). + make_literal_list(Args) -> make_literal_list(Args, []). @@ -611,9 +611,11 @@ make_literal_list([_|_], _) -> make_literal_list([], Acc) -> reverse(Acc). -is_safe_bool_op(Args, Ts) -> - [T1,T2] = get_types(Args, Ts), - t_is_boolean(T1) andalso t_is_boolean(T2). +is_safe_bool_op([LHS, RHS], Ts) -> + LType = raw_type(LHS, Ts), + RType = raw_type(RHS, Ts), + beam_types:is_boolean_type(LType) andalso + beam_types:is_boolean_type(RType). all_same([{H,_}|T]) -> all(fun({E,_}) -> E =:= H end, T). @@ -626,7 +628,7 @@ eval_bif(#b_set{op={bif,Bif},args=Args}=I, Ts) -> true -> case make_literal_list(Args) of none -> - case get_types(Args, Ts) of + case normalized_types(Args, Ts) of [any] -> I; [Type] -> @@ -659,10 +661,9 @@ simplify_arg(#b_var{}=Arg0, Sub, Ts) -> #b_literal{}=LitArg -> LitArg; #b_var{}=Arg -> - Type = get_type(Arg, Ts), - case get_literal_from_type(Type) of - none -> Arg; - #b_literal{}=Lit -> Lit + case beam_types:get_singleton_value(raw_type(Arg, Ts)) of + {ok, Val} -> #b_literal{val=Val}; + error -> Arg end end; simplify_arg(#b_remote{mod=Mod,name=Name}=Rem, Sub, Ts) -> @@ -701,7 +702,7 @@ opt_terminator(#b_br{bool=#b_var{}}=Br, Ts, Ds) -> opt_terminator(#b_switch{arg=#b_literal{}}=Sw, _Ts, _Ds) -> beam_ssa:normalize(Sw); opt_terminator(#b_switch{arg=#b_var{}=V}=Sw, Ts, Ds) -> - case get_type(V, Ts) of + case normalized_type(V, Ts) of any -> beam_ssa:normalize(Sw); Type -> @@ -717,7 +718,7 @@ opt_switch(#b_switch{fail=Fail,list=List0}=Sw0, Type, Ts, Ds) -> #t_integer{elements={_,_}=Range} -> simplify_switch_int(Sw1, Range); #t_atom{elements=[_|_]} -> - case t_is_boolean(Type) of + case beam_types:is_boolean_type(Type) of true -> #b_br{} = Br = simplify_switch_bool(Sw1, Ts, Ds), opt_terminator(Br, Ts, Ds); @@ -731,7 +732,7 @@ opt_switch(#b_switch{fail=Fail,list=List0}=Sw0, Type, Ts, Ds) -> prune_switch_list([{_,Fail}|T], Fail, Type, Ts) -> prune_switch_list(T, Fail, Type, Ts); prune_switch_list([{Arg,_}=Pair|T], Fail, Type, Ts) -> - case meet(get_type(Arg, Ts), Type) of + case beam_types:meet(raw_type(Arg, Ts), Type) of none -> %% Different types. This value can never match. prune_switch_list(T, Fail, Type, Ts); @@ -740,82 +741,91 @@ prune_switch_list([{Arg,_}=Pair|T], Fail, Type, Ts) -> end; prune_switch_list([], _, _, _) -> []. -update_successors(#b_br{bool=#b_literal{val=true},succ=S}, Ts, D) -> - update_successor(S, Ts, D); -update_successors(#b_br{bool=#b_var{}=Bool,succ=Succ,fail=Fail}, Ts0, D0) -> - case cerl_sets:is_element(Bool, D0#d.once) of - true -> - %% This variable is defined in this block and is only - %% referenced by this br terminator. Therefore, there is - %% no need to include it in the type database passed on to - %% the successors of this block. - Ts = maps:remove(Bool, Ts0), - {SuccTs,FailTs} = infer_types_br(Bool, Ts, D0), - D = update_successor(Fail, FailTs, D0), - update_successor(Succ, SuccTs, D); - false -> - {SuccTs,FailTs} = infer_types_br(Bool, Ts0, D0), - D = update_successor_bool(Bool, false, Fail, FailTs, D0), - update_successor_bool(Bool, true, Succ, SuccTs, D) - end; -update_successors(#b_switch{arg=#b_var{}=V,fail=Fail,list=List}, Ts, D0) -> - case cerl_sets:is_element(V, D0#d.once) of - true -> - %% This variable is defined in this block and is only - %% referenced by this switch terminator. Therefore, there is - %% no need to include it in the type database passed on to - %% the successors of this block. - D = update_successor(Fail, Ts, D0), - F = fun({Val,S}, A) -> - SuccTs0 = infer_types_switch(V, Val, Ts, D), - SuccTs = maps:remove(V, SuccTs0), - update_successor(S, SuccTs, A) - end, - foldl(F, D, List); - false -> - %% V can not be equal to any of the values in List at the fail - %% block. - FailTs = subtract_sw_list(V, List, Ts), - D = update_successor(Fail, FailTs, D0), - F = fun({Val,S}, A) -> - SuccTs = infer_types_switch(V, Val, Ts, D), - update_successor(S, SuccTs, A) - end, - foldl(F, D, List) +update_successors(#b_br{bool=#b_literal{val=true},succ=Succ}=Last, Ts, D0) -> + {Last, update_successor(Succ, Ts, D0)}; +update_successors(#b_br{bool=#b_var{}=Bool,succ=Succ,fail=Fail}=Last0, + Ts, D0) -> + UsedOnce = cerl_sets:is_element(Bool, D0#d.once), + case infer_types_br(Bool, Ts, UsedOnce, D0) of + {#{}=SuccTs, #{}=FailTs} -> + D1 = update_successor(Succ, SuccTs, D0), + D = update_successor(Fail, FailTs, D1), + {Last0, D}; + {#{}=SuccTs, none} -> + Last = Last0#b_br{bool=#b_literal{val=true},fail=Succ}, + {Last, update_successor(Succ, SuccTs, D0)}; + {none, #{}=FailTs} -> + Last = Last0#b_br{bool=#b_literal{val=true},succ=Fail}, + {Last, update_successor(Fail, FailTs, D0)} end; -update_successors(#b_ret{arg=Arg}, Ts, D) -> - FuncId = D#d.func_id, - case D#d.ds of - #{ Arg := #b_set{op=call,args=[FuncId | _]} } -> - %% Returning a call to ourselves doesn't affect our own return - %% type. - D; +update_successors(#b_switch{arg=#b_var{}=V,fail=Fail0,list=List0}=Last0, + Ts, D0) -> + UsedOnce = cerl_sets:is_element(V, D0#d.once), + + {List1, D1} = update_switch(List0, V, Ts, UsedOnce, [], D0), + FailTs = update_switch_failure(V, List0, Ts, UsedOnce, D1), + + case FailTs of + none -> + %% The fail block is unreachable; swap it with one of the choices. + [{_, Fail} | List] = List1, + Last = Last0#b_switch{fail=Fail,list=List}, + {Last, D1}; #{} -> - RetType = join([get_type(Arg, Ts) | D#d.ret_type]), - D#d{ret_type=[RetType]} - end. + D = update_successor(Fail0, FailTs, D1), + Last = Last0#b_switch{list=List1}, + {Last, D} + end; +update_successors(#b_ret{arg=Arg}=Last, Ts, D0) -> + FuncId = D0#d.func_id, + D = case D0#d.ds of + #{ Arg := #b_set{op=call,args=[FuncId | _]} } -> + %% Returning a call to ourselves doesn't affect our own return + %% type. + D0; + #{} -> + RetType = beam_types:join([raw_type(Arg, Ts) | D0#d.ret_type]), + D0#d{ret_type=[RetType]} + end, + {Last, D}. + +update_switch([{Val, Lbl}=Sw | List], V, Ts, UsedOnce, Acc, D0) -> + case infer_types_switch(V, Val, Ts, UsedOnce, D0) of + none -> + update_switch(List, V, Ts, UsedOnce, Acc, D0); + SwTs -> + D = update_successor(Lbl, SwTs, D0), + update_switch(List, V, Ts, UsedOnce, [Sw | Acc], D) + end; +update_switch([], _V, _Ts, _UsedOnce, Acc, D) -> + {reverse(Acc), D}. -subtract_sw_list(V, List, Ts) -> - Ts#{ V := sub_sw_list_1(get_type(V, Ts), List, Ts) }. +update_switch_failure(V, List, Ts, UsedOnce, D) -> + case sub_sw_list_1(raw_type(V, Ts), List, Ts) of + none -> + none; + FailType -> + case beam_types:get_singleton_value(FailType) of + {ok, Value} -> + %% This is the only possible value at the fail label, so we + %% can infer types as if we matched it directly. + Lit = #b_literal{val=Value}, + infer_types_switch(V, Lit, Ts, UsedOnce, D); + error when UsedOnce -> + ts_remove_var(V, Ts); + error -> + Ts + end + end. sub_sw_list_1(Type, [{Val,_}|T], Ts) -> - ValType = get_type(Val, Ts), - sub_sw_list_1(subtract(Type, ValType), T, Ts); + ValType = raw_type(Val, Ts), + sub_sw_list_1(beam_types:subtract(Type, ValType), T, Ts); sub_sw_list_1(Type, [], _Ts) -> Type. -update_successor_bool(#b_var{}=Var, BoolValue, S, Ts, D) -> - case t_is_boolean(get_type(Var, Ts)) of - true -> - update_successor(S, Ts#{Var:=t_atom(BoolValue)}, D); - false -> - %% The `br` terminator is preceeded by an instruction that - %% does not produce a boolean value, such a `new_try_tag`. - update_successor(S, Ts, D) - end. - -update_successor(?BADARG_BLOCK, _Ts, #d{}=D) -> - %% We KNOW that no variables are used in the ?BADARG_BLOCK, +update_successor(?EXCEPTION_BLOCK, _Ts, #d{}=D) -> + %% We KNOW that no variables are used in the ?EXCEPTION_BLOCK, %% so there is no need to update the type information. That %% can be a huge timesaver for huge functions. D; @@ -828,247 +838,102 @@ update_successor(S, Ts0, #d{ls=Ls}=D) -> D#d{ls=Ls#{S=>Ts0}} end. -update_types(#b_set{op=Op,dst=Dst,args=Args}, Ts, Ds) -> - T = type(Op, Args, Ts, Ds), +update_types(#b_set{op=Op,dst=Dst,anno=Anno,args=Args}, Ts, Ds) -> + T = type(Op, Args, Anno, Ts, Ds), Ts#{Dst=>T}. -type(phi, Args, Ts, _Ds) -> - Types = [get_type(A, Ts) || {A,_} <- Args], - join(Types); -type({bif,'band'}, Args, Ts, _Ds) -> - band_type(Args, Ts); -type({bif,Bif}, Args, Ts, _Ds) -> - case bif_type(Bif, Args) of - number -> - arith_op_type(Args, Ts); - Type -> - Type +type(phi, Args, _Anno, Ts, _Ds) -> + Types = [raw_type(A, Ts) || {A,_} <- Args], + beam_types:join(Types); +type({bif,Bif}, Args, _Anno, Ts, _Ds) -> + ArgTypes = normalized_types(Args, Ts), + {RetType, _, _} = beam_call_types:types(erlang, Bif, ArgTypes), + RetType; +type(bs_init, _Args, _Anno, _Ts, _Ds) -> + #t_bitstring{}; +type(bs_extract, [Ctx], _Anno, _Ts, Ds) -> + #b_set{op=bs_match,args=Args} = map_get(Ctx, Ds), + bs_match_type(Args); +type(bs_match, _Args, _Anno, _Ts, _Ds) -> + #t_bs_context{}; +type(bs_get_tail, _Args, _Anno, _Ts, _Ds) -> + #t_bitstring{}; +type(call, [#b_local{} | _Args], Anno, _Ts, _Ds) -> + case Anno of + #{ result_type := Type } -> Type; + #{} -> any end; -type(bs_init, _Args, _Ts, _Ds) -> - {binary, 1}; -type(bs_extract, [Ctx], Ts, _Ds) -> - #t_bs_match{type=Type} = get_type(Ctx, Ts), - Type; -type(bs_match, Args, _Ts, _Ds) -> - #t_bs_match{type=bs_match_type(Args)}; -type(bs_get_tail, _Args, _Ts, _Ds) -> - {binary, 1}; type(call, [#b_remote{mod=#b_literal{val=Mod}, - name=#b_literal{val=Name}}|Args], Ts, _Ds) -> - case {Mod,Name,Args} of - {erlang,setelement,[Pos,Tuple,Arg]} -> - case {get_type(Pos, Ts),get_type(Tuple, Ts)} of - {#t_integer{elements={Index,Index}}, - #t_tuple{elements=Es0,size=Size}=T} -> - %% This is an exact index, update the type of said element - %% or return 'none' if it's known to be out of bounds. - Es = set_element_type(Index, get_type(Arg, Ts), Es0), - case T#t_tuple.exact of - false -> - T#t_tuple{size=max(Index, Size),elements=Es}; - true when Index =< Size -> - T#t_tuple{elements=Es}; - true -> - none - end; - {#t_integer{elements={Min,_}}=IntType, - #t_tuple{elements=Es0,size=Size}=T} -> - %% Remove type information for all indices that - %% falls into the range of the integer. - Es = remove_element_info(IntType, Es0), - case T#t_tuple.exact of - false -> - T#t_tuple{elements=Es,size=max(Min, Size)}; - true when Min =< Size -> - T#t_tuple{elements=Es,size=Size}; - true -> - none - end; - {_,#t_tuple{}=T} -> - %% Position unknown, so we have to discard all element - %% information. - T#t_tuple{elements=#{}}; - {#t_integer{elements={Min,_Max}},_} -> - #t_tuple{size=Min}; - {_,_} -> - #t_tuple{} - end; - {erlang,'++',[LHS,RHS]} -> - LType = get_type(LHS, Ts), - RType = get_type(RHS, Ts), - case LType =:= cons orelse RType =:= cons of - true -> - cons; - false -> - %% `[] ++ RHS` yields RHS, even if RHS is not a list. - join(list, RType) - end; - {erlang,'--',[_,_]} -> - list; - {lists,F,Args} -> - Types = get_types(Args, Ts), - lists_function_type(F, Types); - {math,_,_} -> - case is_math_bif(Name, length(Args)) of - false -> any; - true -> float - end; - {_,_,_} -> - case erl_bifs:is_exit_bif(Mod, Name, length(Args)) of - true -> none; - false -> any - end - end; -type(get_tuple_element, [Tuple, Offset], Ts, _Ds) -> - #t_tuple{size=Size,elements=Es} = get_type(Tuple, Ts), + name=#b_literal{val=Name}}|Args], _Anno, Ts, _Ds) -> + ArgTypes = normalized_types(Args, Ts), + {RetType, _, _} = beam_call_types:types(Mod, Name, ArgTypes), + RetType; +type(get_tuple_element, [Tuple, Offset], _Anno, Ts, _Ds) -> + #t_tuple{size=Size,elements=Es} = normalized_type(Tuple, Ts), #b_literal{val=N} = Offset, true = Size > N, %Assertion. - get_element_type(N + 1, Es); -type(is_nonempty_list, [_], _Ts, _Ds) -> - t_boolean(); -type(is_tagged_tuple, [_,#b_literal{},#b_literal{}], _Ts, _Ds) -> - t_boolean(); -type(put_map, _Args, _Ts, _Ds) -> - map; -type(put_list, _Args, _Ts, _Ds) -> + beam_types:get_element_type(N + 1, Es); +type(is_nonempty_list, [_], _Anno, _Ts, _Ds) -> + beam_types:make_boolean(); +type(is_tagged_tuple, [_,#b_literal{},#b_literal{}], _Anno, _Ts, _Ds) -> + beam_types:make_boolean(); +type(make_fun, [#b_local{arity=TotalArity}|Env], _Anno, _Ts, _Ds) -> + #t_fun{arity=TotalArity-length(Env)}; +type(put_map, _Args, _Anno, _Ts, _Ds) -> + #t_map{}; +type(put_list, _Args, _Anno, _Ts, _Ds) -> cons; -type(put_tuple, Args, Ts, _Ds) -> +type(put_tuple, Args, _Anno, Ts, _Ds) -> {Es, _} = foldl(fun(Arg, {Es0, Index}) -> - Type = get_type(Arg, Ts), - Es = set_element_type(Index, Type, Es0), - {Es, Index + 1} + Type = raw_type(Arg, Ts), + Es = beam_types:set_element_type(Index, Type, Es0), + {Es, Index + 1} end, {#{}, 1}, Args), #t_tuple{exact=true,size=length(Args),elements=Es}; -type(succeeded, [#b_var{}=Src], Ts, Ds) -> +type(succeeded, [#b_var{}=Src], _Anno, Ts, _Ds) + when map_get(Src, Ts) =:= none -> + beam_types:make_atom(false); +type(succeeded, [#b_var{}=Src], _Anno, Ts, Ds) -> case maps:get(Src, Ds) of #b_set{op={bif,Bif},args=BifArgs} -> - Types = get_types(BifArgs, Ts), - case {Bif,Types} of - {BoolOp,[T1,T2]} when BoolOp =:= 'and'; BoolOp =:= 'or' -> - case t_is_boolean(T1) andalso t_is_boolean(T2) of - true -> t_atom(true); - false -> t_boolean() - end; - {byte_size,[{binary,_}]} -> - t_atom(true); - {bit_size,[{binary,_}]} -> - t_atom(true); - {map_size,[map]} -> - t_atom(true); - {'not',[Type]} -> - case t_is_boolean(Type) of - true -> t_atom(true); - false -> t_boolean() - end; - {size,[{binary,_}]} -> - t_atom(true); - {tuple_size,[#t_tuple{}]} -> - t_atom(true); - {_,_} -> - t_boolean() + ArgTypes = normalized_types(BifArgs, Ts), + case beam_call_types:will_succeed(erlang, Bif, ArgTypes) of + yes -> beam_types:make_atom(true); + no -> beam_types:make_atom(false); + maybe -> beam_types:make_boolean() + end; + #b_set{op=call,args=[#b_remote{mod=#b_literal{val=Mod}, + name=#b_literal{val=Func}} | + CallArgs]} -> + ArgTypes = normalized_types(CallArgs, Ts), + case beam_call_types:will_succeed(Mod, Func, ArgTypes) of + yes -> beam_types:make_atom(true); + no -> beam_types:make_atom(false); + maybe -> beam_types:make_boolean() end; #b_set{op=get_hd} -> - t_atom(true); + beam_types:make_atom(true); #b_set{op=get_tl} -> - t_atom(true); + beam_types:make_atom(true); #b_set{op=get_tuple_element} -> - t_atom(true); + beam_types:make_atom(true); + #b_set{op=put_tuple} -> + beam_types:make_atom(true); #b_set{op=wait} -> - t_atom(false); + beam_types:make_atom(false); #b_set{} -> - t_boolean() + beam_types:make_boolean() end; -type(succeeded, [#b_literal{}], _Ts, _Ds) -> - t_atom(true); -type(_, _, _, _) -> any. - -arith_op_type(Args, Ts) -> - Types = get_types(Args, Ts), - foldl(fun(#t_integer{}, unknown) -> t_integer(); - (#t_integer{}, number) -> number; - (#t_integer{}, float) -> float; - (#t_integer{}, #t_integer{}) -> t_integer(); - (float, unknown) -> float; - (float, #t_integer{}) -> float; - (float, number) -> float; - (number, unknown) -> number; - (number, #t_integer{}) -> number; - (number, float) -> float; - (any, _) -> number; - (Same, Same) -> Same; - (_, _) -> none - end, unknown, Types). - -lists_function_type(F, Types) -> - case {F,Types} of - %% Functions that return booleans. - {all,[_,_]} -> - t_boolean(); - {any,[_,_]} -> - t_boolean(); - {keymember,[_,_,_]} -> - t_boolean(); - {member,[_,_]} -> - t_boolean(); - {prefix,[_,_]} -> - t_boolean(); - {suffix,[_,_]} -> - t_boolean(); - - %% Functions that return lists. - {dropwhile,[_,_]} -> - list; - {duplicate,[_,_]} -> - list; - {filter,[_,_]} -> - list; - {flatten,[_]} -> - list; - {map,[_Fun,List]} -> - same_length_type(List); - {MapFold,[_Fun,_Acc,List]} when MapFold =:= mapfoldl; - MapFold =:= mapfoldr -> - #t_tuple{size=2,exact=true, - elements=#{1=>same_length_type(List)}}; - {partition,[_,_]} -> - t_two_tuple(list, list); - {reverse,[List]} -> - same_length_type(List); - {sort,[List]} -> - same_length_type(List); - {splitwith,[_,_]} -> - t_two_tuple(list, list); - {takewhile,[_,_]} -> - list; - {unzip,[List]} -> - ListType = same_length_type(List), - t_two_tuple(ListType, ListType); - {usort,[List]} -> - same_length_type(List); - {zip,[_,_]} -> - list; - {zipwith,[_,_,_]} -> - list; - {_,_} -> - any - end. - -%% For a lists function that return a list of the same -%% length as the input list, return the type of the list. -same_length_type(cons) -> cons; -same_length_type(nil) -> nil; -same_length_type(_) -> list. - -t_two_tuple(Type1, Type2) -> - #t_tuple{size=2,exact=true, - elements=#{1=>Type1,2=>Type2}}. +type(succeeded, [#b_literal{}], _Anno, _Ts, _Ds) -> + beam_types:make_atom(true); +type(_, _, _, _, _) -> any. %% will_succeed(TestOperation, Type) -> yes|no|maybe. %% Test whether TestOperation applied to an argument of type Type %% will succeed. Return yes, no, or maybe. %% -%% Type is a type as described in the comment for verified_type/1 at -%% the very end of this file, but it will *never* be 'any'. +%% Type can be any type as described in beam_types.hrl, but it must *never* be +%% any. will_succeed(is_atom, Type) -> case Type of @@ -1077,13 +942,13 @@ will_succeed(is_atom, Type) -> end; will_succeed(is_binary, Type) -> case Type of - {binary,U} when U rem 8 =:= 0 -> yes; - {binary,_} -> maybe; + #t_bitstring{unit=U} when U rem 8 =:= 0 -> yes; + #t_bitstring{} -> maybe; _ -> no end; will_succeed(is_bitstring, Type) -> case Type of - {binary,_} -> yes; + #t_bitstring{} -> yes; _ -> no end; will_succeed(is_boolean, Type) -> @@ -1091,7 +956,7 @@ will_succeed(is_boolean, Type) -> #t_atom{elements=any} -> maybe; #t_atom{elements=Es} -> - case t_is_boolean(Type) of + case beam_types:is_boolean_type(Type) of true -> yes; false -> @@ -1109,6 +974,11 @@ will_succeed(is_float, Type) -> number -> maybe; _ -> no end; +will_succeed(is_function, Type) -> + case Type of + #t_fun{} -> yes; + _ -> no + end; will_succeed(is_integer, Type) -> case Type of #t_integer{} -> yes; @@ -1123,7 +993,7 @@ will_succeed(is_list, Type) -> end; will_succeed(is_map, Type) -> case Type of - map -> yes; + #t_map{} -> yes; _ -> no end; will_succeed(is_number, Type) -> @@ -1140,35 +1010,12 @@ will_succeed(is_tuple, Type) -> end; will_succeed(_, _) -> maybe. - -band_type([Other,#b_literal{val=Int}], Ts) when is_integer(Int) -> - band_type_1(Int, Other, Ts); -band_type([_,_], _) -> t_integer(). - -band_type_1(Int, OtherSrc, Ts) -> - Type = band_type_2(Int, 0), - OtherType = get_type(OtherSrc, Ts), - meet(Type, OtherType). - -band_type_2(N, Bits) when Bits < 64 -> - case 1 bsl Bits of - P when P =:= N + 1 -> - t_integer(0, N); - P when P > N + 1 -> - t_integer(); - _ -> - band_type_2(N, Bits+1) - end; -band_type_2(_, _) -> - %% Negative or large positive number. Give up. - t_integer(). - bs_match_type([#b_literal{val=Type}|Args]) -> bs_match_type(Type, Args). bs_match_type(binary, Args) -> [_,_,_,#b_literal{val=U}] = Args, - {binary,U}; + #t_bitstring{unit=U}; bs_match_type(float, _) -> float; bs_match_type(integer, Args) -> @@ -1180,24 +1027,24 @@ bs_match_type(integer, Args) -> NumBits = Size * Unit, case member(unsigned, Flags) of true -> - t_integer(0, (1 bsl NumBits)-1); + beam_types:make_integer(0, (1 bsl NumBits)-1); false -> %% Signed integer. Don't bother. - t_integer() + #t_integer{} end; [_|_] -> - t_integer() + #t_integer{} end; bs_match_type(skip, _) -> any; bs_match_type(string, _) -> any; bs_match_type(utf8, _) -> - ?UNICODE_INT; + beam_types:make_integer(0, ?UNICODE_MAX); bs_match_type(utf16, _) -> - ?UNICODE_INT; + beam_types:make_integer(0, ?UNICODE_MAX); bs_match_type(utf32, _) -> - ?UNICODE_INT. + beam_types:make_integer(0, ?UNICODE_MAX). simplify_switch_atom(#t_atom{elements=Atoms}, #b_switch{list=List0}=Sw) -> case sort([A || {#b_literal{val=A},_} <- List0]) of @@ -1229,14 +1076,14 @@ eq_ranges(_, _, _) -> false. simplify_is_record(I, #t_tuple{exact=Exact, size=Size, elements=Es}, - RecSize, RecTag, Ts) -> + RecSize, #b_literal{val=TagVal}=RecTag, Ts) -> TagType = maps:get(1, Es, any), - TagMatch = case get_literal_from_type(TagType) of - #b_literal{}=RecTag -> yes; - #b_literal{} -> no; - none -> + TagMatch = case beam_types:get_singleton_value(TagType) of + {ok, TagVal} -> yes; + {ok, _} -> no; + error -> %% Is it at all possible for the tag to match? - case meet(get_type(RecTag, Ts), TagType) of + case beam_types:meet(raw_type(RecTag, Ts), TagType) of none -> no; _ -> maybe end @@ -1266,7 +1113,7 @@ simplify_switch_bool(#b_switch{arg=B,fail=Fail,list=List0}, Ts, Ds) -> simplify_not(#b_br{bool=#b_var{}=V,succ=Succ,fail=Fail}=Br0, Ts, Ds) -> case Ds of #{V:=#b_set{op={bif,'not'},args=[Bool]}} -> - case t_is_boolean(get_type(Bool, Ts)) of + case beam_types:is_boolean_type(raw_type(Bool, Ts)) of true -> Br = Br0#b_br{bool=Bool,succ=Fail,fail=Succ}, beam_ssa:normalize(Br); @@ -1334,40 +1181,18 @@ used_once_last_uses([V|Vs], L, Uses) -> end; used_once_last_uses([], _, Uses) -> Uses. +normalized_types(Values, Ts) -> + [normalized_type(Val, Ts) || Val <- Values]. -get_types(Values, Ts) -> - [get_type(Val, Ts) || Val <- Values]. --spec get_type(beam_ssa:value(), type_db()) -> type(). +normalized_type(V, Ts) -> + beam_types:normalize(raw_type(V, Ts)). -get_type(#b_var{}=V, Ts) -> - #{V:=T} = Ts, - T; -get_type(#b_literal{val=Val}, _Ts) -> - if - is_atom(Val) -> - t_atom(Val); - is_float(Val) -> - float; - is_integer(Val) -> - t_integer(Val); - is_list(Val), Val =/= [] -> - cons; - is_map(Val) -> - map; - Val =:= {} -> - #t_tuple{exact=true}; - is_tuple(Val) -> - {Es, _} = foldl(fun(E, {Es0, Index}) -> - Type = get_type(#b_literal{val=E}, #{}), - Es = set_element_type(Index, Type, Es0), - {Es, Index + 1} - end, {#{}, 1}, tuple_to_list(Val)), - #t_tuple{exact=true,size=tuple_size(Val),elements=Es}; - Val =:= [] -> - nil; - true -> - any - end. +-spec raw_type(beam_ssa:value(), type_db()) -> type(). + +raw_type(#b_literal{val=Value}, _Ts) -> + beam_types:make_type_from_value(Value); +raw_type(V, Ts) -> + map_get(V, Ts). %% infer_types(Var, Types, #d{}) -> {SuccTypes,FailTypes} %% Looking at the expression that defines the variable Var, infer @@ -1390,10 +1215,107 @@ get_type(#b_literal{val=Val}, _Ts) -> %% 'cons' would give 'nil' as the only possible type. The result of the %% subtraction for L will be added to FailTypes. -infer_types_br(#b_var{}=V, Ts, #d{ds=Ds}) -> +infer_types_br(#b_var{}=V, Ts, UsedOnce, #d{ds=Ds}) -> #{V:=#b_set{op=Op,args=Args}} = Ds, - PosTypes0 = infer_type(Op, Args, Ds), - NegTypes0 = infer_type_negative(Op, Args, Ds), + + {PosTypes, NegTypes} = infer_type(Op, Args, Ts, Ds), + + SuccTs0 = meet_types(PosTypes, Ts), + FailTs0 = subtract_types(NegTypes, Ts), + + case UsedOnce of + true -> + %% The branch variable is defined in this block and is only + %% referenced by this terminator. Therefore, there is no need to + %% include it in the type database passed on to the successors of + %% of this block. + SuccTs = ts_remove_var(V, SuccTs0), + FailTs = ts_remove_var(V, FailTs0), + {SuccTs, FailTs}; + false -> + SuccTs = infer_br_value(V, true, SuccTs0), + FailTs = infer_br_value(V, false, FailTs0), + {SuccTs, FailTs} + end. + +infer_br_value(_V, _Bool, none) -> + none; +infer_br_value(V, Bool, NewTs) -> + #{ V := T } = NewTs, + case beam_types:is_boolean_type(T) of + true -> + NewTs#{ V := beam_types:make_atom(Bool) }; + false -> + %% V is a try/catch tag or similar, leave it alone. + NewTs + end. + +infer_types_switch(V, Lit, Ts0, UsedOnce, #d{ds=Ds}) -> + {PosTypes, _} = infer_type({bif,'=:='}, [V, Lit], Ts0, Ds), + Ts = meet_types(PosTypes, Ts0), + case UsedOnce of + true -> ts_remove_var(V, Ts); + false -> Ts + end. + +ts_remove_var(_V, none) -> none; +ts_remove_var(V, Ts) -> maps:remove(V, Ts). + +infer_type(succeeded, [#b_var{}=Src], Ts, Ds) -> + #b_set{op=Op,args=Args} = maps:get(Src, Ds), + infer_success_type(Op, Args, Ts, Ds); + +%% Type tests are handled separately from other BIFs as we're inferring types +%% based on their result, so we know that subtraction is safe even if we're +%% not branching on 'succeeded'. +infer_type(is_tagged_tuple, [#b_var{}=Src,#b_literal{val=Size}, + #b_literal{}=Tag], _Ts, _Ds) -> + Es = beam_types:set_element_type(1, raw_type(Tag, #{}), #{}), + T = {Src,#t_tuple{exact=true,size=Size,elements=Es}}, + {[T], [T]}; +infer_type(is_nonempty_list, [#b_var{}=Src], _Ts, _Ds) -> + T = {Src,cons}, + {[T], [T]}; +infer_type({bif,is_atom}, [Arg], _Ts, _Ds) -> + T = {Arg, #t_atom{}}, + {[T], [T]}; +infer_type({bif,is_binary}, [Arg], _Ts, _Ds) -> + T = {Arg, #t_bitstring{unit=8}}, + {[T], [T]}; +infer_type({bif,is_bitstring}, [Arg], _Ts, _Ds) -> + T = {Arg, #t_bitstring{}}, + {[T], [T]}; +infer_type({bif,is_boolean}, [Arg], _Ts, _Ds) -> + T = {Arg, beam_types:make_boolean()}, + {[T], [T]}; +infer_type({bif,is_float}, [Arg], _Ts, _Ds) -> + T = {Arg, float}, + {[T], [T]}; +infer_type({bif,is_integer}, [Arg], _Ts, _Ds) -> + T = {Arg, #t_integer{}}, + {[T], [T]}; +infer_type({bif,is_list}, [Arg], _Ts, _Ds) -> + T = {Arg, list}, + {[T], [T]}; +infer_type({bif,is_map}, [Arg], _Ts, _Ds) -> + T = {Arg, #t_map{}}, + {[T], [T]}; +infer_type({bif,is_number}, [Arg], _Ts, _Ds) -> + T = {Arg, number}, + {[T], [T]}; +infer_type({bif,is_tuple}, [Arg], _Ts, _Ds) -> + T = {Arg, #t_tuple{}}, + {[T], [T]}; +infer_type({bif,'=:='}, [#b_var{}=LHS,#b_var{}=RHS], Ts, _Ds) -> + %% As an example, assume that L1 is known to be 'list', and L2 is + %% known to be 'cons'. Then if 'L1 =:= L2' evaluates to 'true', it can + %% be inferred that L1 is 'cons' (the meet of 'cons' and 'list'). + LType = raw_type(LHS, Ts), + RType = raw_type(RHS, Ts), + Type = beam_types:meet(LType, RType), + + PosTypes = [{V,Type} || {V, OrigType} <- [{LHS, LType}, {RHS, RType}], + OrigType =/= Type], %% We must be careful with types inferred from '=:='. %% @@ -1404,39 +1326,39 @@ infer_types_br(#b_var{}=V, Ts, #d{ds=Ds}) -> %% %% However, it is safe to subtract a type inferred from '=:=' if %% it is single-valued, e.g. if it is [] or the atom 'true'. + NegTypes = case beam_types:is_singleton_type(Type) of + true -> PosTypes; + false -> [] + end, - EqTypes = infer_eq_type(Op, Args, Ts, Ds), - NegTypes1 = [P || {_,T}=P <- EqTypes, is_singleton_type(T)], - - PosTypes = EqTypes ++ PosTypes0, - SuccTs = meet_types(PosTypes, Ts), - - NegTypes = NegTypes0 ++ NegTypes1, - FailTs = subtract_types(NegTypes, Ts), - - {SuccTs,FailTs}. - -infer_types_switch(V, Lit, Ts, #d{ds=Ds}) -> - Types = infer_eq_type({bif,'=:='}, [V, Lit], Ts, Ds), - meet_types(Types, Ts). - -infer_eq_type({bif,'=:='}, [#b_var{}=Src,#b_literal{}=Lit], Ts, Ds) -> + {PosTypes, NegTypes}; +infer_type({bif,'=:='}, [#b_var{}=Src,#b_literal{}=Lit], Ts, Ds) -> Def = maps:get(Src, Ds), - Type = get_type(Lit, Ts), - [{Src,Type} | infer_eq_lit(Def, Lit)]; -infer_eq_type({bif,'=:='}, [#b_var{}=Arg0,#b_var{}=Arg1], Ts, _Ds) -> - %% As an example, assume that L1 is known to be 'list', and L2 is - %% known to be 'cons'. Then if 'L1 =:= L2' evaluates to 'true', it can - %% be inferred that L1 is 'cons' (the meet of 'cons' and 'list'). - Type0 = get_type(Arg0, Ts), - Type1 = get_type(Arg1, Ts), - Type = meet(Type0, Type1), - [{V,MeetType} || - {V,OrigType,MeetType} <- - [{Arg0,Type0,Type},{Arg1,Type1,Type}], - OrigType =/= MeetType]; -infer_eq_type(_Op, _Args, _Ts, _Ds) -> - []. + Type = raw_type(Lit, Ts), + EqLitTypes = infer_eq_lit(Def, Lit), + PosTypes = [{Src,Type} | EqLitTypes], + {PosTypes, EqLitTypes}; +infer_type(_Op, _Args, _Ts, _Ds) -> + {[], []}. + +infer_success_type({bif,Op}, Args, Ts, _Ds) -> + ArgTypes = normalized_types(Args, Ts), + + {_, PosTypes0, CanSubtract} = beam_call_types:types(erlang, Op, ArgTypes), + PosTypes = [T || {#b_var{},_}=T <- zip(Args, PosTypes0)], + + case CanSubtract of + true -> {PosTypes, PosTypes}; + false -> {PosTypes, []} + end; +infer_success_type(call, [#b_var{}=Fun|Args], _Ts, _Ds) -> + T = {Fun, #t_fun{arity=length(Args)}}, + {[T], []}; +infer_success_type(bs_start_match, [#b_var{}=Bin], _Ts, _Ds) -> + T = {Bin,#t_bitstring{}}, + {[T], [T]}; +infer_success_type(_Op, _Args, _Ts, _Ds) -> + {[], []}. infer_eq_lit(#b_set{op={bif,tuple_size},args=[#b_var{}=Tuple]}, #b_literal{val=Size}) when is_integer(Size) -> @@ -1445,178 +1367,11 @@ infer_eq_lit(#b_set{op=get_tuple_element, args=[#b_var{}=Tuple,#b_literal{val=N}]}, #b_literal{}=Lit) -> Index = N + 1, - Es = set_element_type(Index, get_type(Lit, #{}), #{}), + Es = beam_types:set_element_type(Index, raw_type(Lit, #{}), #{}), [{Tuple,#t_tuple{size=Index,elements=Es}}]; -infer_eq_lit(_, _) -> []. - -infer_type_negative(Op, Args, Ds) -> - case is_negative_inference_safe(Op, Args) of - true -> - infer_type(Op, Args, Ds); - false -> - [] - end. - -%% Conservative list of instructions for which negative -%% inference is safe. -is_negative_inference_safe(is_nonempty_list, _Args) -> true; -is_negative_inference_safe(_, _) -> false. - -infer_type({bif,element}, [#b_literal{val=Pos},#b_var{}=Tuple], _Ds) -> - if - is_integer(Pos), 1 =< Pos -> - [{Tuple,#t_tuple{size=Pos}}]; - true -> - [] - end; -infer_type({bif,element}, [#b_var{}=Position,#b_var{}=Tuple], _Ds) -> - [{Position,t_integer()},{Tuple,#t_tuple{}}]; -infer_type({bif,Bif}, [#b_var{}=Src]=Args, _Ds) -> - case inferred_bif_type(Bif, Args) of - any -> []; - T -> [{Src,T}] - end; -infer_type({bif,binary_part}, [#b_var{}=Src,_], _Ds) -> - [{Src,{binary,8}}]; -infer_type({bif,is_map_key}, [_,#b_var{}=Src], _Ds) -> - [{Src,map}]; -infer_type({bif,map_get}, [_,#b_var{}=Src], _Ds) -> - [{Src,map}]; -infer_type({bif,Bif}, [_,_]=Args, _Ds) -> - case inferred_bif_type(Bif, Args) of - any -> []; - T -> [{A,T} || #b_var{}=A <- Args] - end; -infer_type({bif,binary_part}, [#b_var{}=Src,Pos,Len], _Ds) -> - [{Src,{binary,8}}| - [{V,t_integer()} || #b_var{}=V <- [Pos,Len]]]; -infer_type(bs_start_match, [#b_var{}=Bin], _Ds) -> - [{Bin,{binary,1}}]; -infer_type(is_nonempty_list, [#b_var{}=Src], _Ds) -> - [{Src,cons}]; -infer_type(is_tagged_tuple, [#b_var{}=Src,#b_literal{val=Size}, - #b_literal{}=Tag], _Ds) -> - Es = set_element_type(1, get_type(Tag, #{}), #{}), - [{Src,#t_tuple{exact=true,size=Size,elements=Es}}]; -infer_type(succeeded, [#b_var{}=Src], Ds) -> - #b_set{op=Op,args=Args} = maps:get(Src, Ds), - infer_type(Op, Args, Ds); -infer_type(_Op, _Args, _Ds) -> +infer_eq_lit(_, _) -> []. -%% bif_type(Name, Args) -> Type -%% Return the return type for the guard BIF or operator Name with -%% arguments Args. -%% -%% Note that that the following BIFs are handle elsewhere: -%% -%% band/2 - -bif_type(abs, [_]) -> number; -bif_type(bit_size, [_]) -> t_integer(); -bif_type(byte_size, [_]) -> t_integer(); -bif_type(ceil, [_]) -> t_integer(); -bif_type(float, [_]) -> float; -bif_type(floor, [_]) -> t_integer(); -bif_type(is_map_key, [_,_]) -> t_boolean(); -bif_type(length, [_]) -> t_integer(); -bif_type(map_size, [_]) -> t_integer(); -bif_type(node, []) -> #t_atom{}; -bif_type(node, [_]) -> #t_atom{}; -bif_type(round, [_]) -> t_integer(); -bif_type(size, [_]) -> t_integer(); -bif_type(trunc, [_]) -> t_integer(); -bif_type(tuple_size, [_]) -> t_integer(); -bif_type('bnot', [_]) -> t_integer(); -bif_type('bor', [_,_]) -> t_integer(); -bif_type('bsl', [_,_]) -> t_integer(); -bif_type('bsr', [_,_]) -> t_integer(); -bif_type('bxor', [_,_]) -> t_integer(); -bif_type('div', [_,_]) -> t_integer(); -bif_type('rem', [_,_]) -> t_integer(); -bif_type('/', [_,_]) -> float; -bif_type(Name, Args) -> - Arity = length(Args), - case erl_internal:new_type_test(Name, Arity) orelse - erl_internal:bool_op(Name, Arity) orelse - erl_internal:comp_op(Name, Arity) of - true -> - t_boolean(); - false -> - case erl_internal:arith_op(Name, Arity) of - true -> number; - false -> any - end - end. - -inferred_bif_type(is_atom, [_]) -> t_atom(); -inferred_bif_type(is_binary, [_]) -> {binary,8}; -inferred_bif_type(is_bitstring, [_]) -> {binary,1}; -inferred_bif_type(is_boolean, [_]) -> t_boolean(); -inferred_bif_type(is_float, [_]) -> float; -inferred_bif_type(is_integer, [_]) -> t_integer(); -inferred_bif_type(is_list, [_]) -> list; -inferred_bif_type(is_map, [_]) -> map; -inferred_bif_type(is_number, [_]) -> number; -inferred_bif_type(is_tuple, [_]) -> #t_tuple{}; -inferred_bif_type(abs, [_]) -> number; -inferred_bif_type(bit_size, [_]) -> {binary,1}; -inferred_bif_type('bnot', [_]) -> t_integer(); -inferred_bif_type(byte_size, [_]) -> {binary,1}; -inferred_bif_type(ceil, [_]) -> number; -inferred_bif_type(float, [_]) -> number; -inferred_bif_type(floor, [_]) -> number; -inferred_bif_type(hd, [_]) -> cons; -inferred_bif_type(length, [_]) -> list; -inferred_bif_type(map_size, [_]) -> map; -inferred_bif_type('not', [_]) -> t_boolean(); -inferred_bif_type(round, [_]) -> number; -inferred_bif_type(trunc, [_]) -> number; -inferred_bif_type(tl, [_]) -> cons; -inferred_bif_type(tuple_size, [_]) -> #t_tuple{}; -inferred_bif_type('and', [_,_]) -> t_boolean(); -inferred_bif_type('or', [_,_]) -> t_boolean(); -inferred_bif_type('xor', [_,_]) -> t_boolean(); -inferred_bif_type('band', [_,_]) -> t_integer(); -inferred_bif_type('bor', [_,_]) -> t_integer(); -inferred_bif_type('bsl', [_,_]) -> t_integer(); -inferred_bif_type('bsr', [_,_]) -> t_integer(); -inferred_bif_type('bxor', [_,_]) -> t_integer(); -inferred_bif_type('div', [_,_]) -> t_integer(); -inferred_bif_type('rem', [_,_]) -> t_integer(); -inferred_bif_type('+', [_,_]) -> number; -inferred_bif_type('-', [_,_]) -> number; -inferred_bif_type('*', [_,_]) -> number; -inferred_bif_type('/', [_,_]) -> number; -inferred_bif_type(_, _) -> any. - -is_math_bif(cos, 1) -> true; -is_math_bif(cosh, 1) -> true; -is_math_bif(sin, 1) -> true; -is_math_bif(sinh, 1) -> true; -is_math_bif(tan, 1) -> true; -is_math_bif(tanh, 1) -> true; -is_math_bif(acos, 1) -> true; -is_math_bif(acosh, 1) -> true; -is_math_bif(asin, 1) -> true; -is_math_bif(asinh, 1) -> true; -is_math_bif(atan, 1) -> true; -is_math_bif(atanh, 1) -> true; -is_math_bif(erf, 1) -> true; -is_math_bif(erfc, 1) -> true; -is_math_bif(exp, 1) -> true; -is_math_bif(log, 1) -> true; -is_math_bif(log2, 1) -> true; -is_math_bif(log10, 1) -> true; -is_math_bif(sqrt, 1) -> true; -is_math_bif(atan2, 2) -> true; -is_math_bif(pow, 2) -> true; -is_math_bif(ceil, 1) -> true; -is_math_bif(floor, 1) -> true; -is_math_bif(fmod, 2) -> true; -is_math_bif(pi, 0) -> true; -is_math_bif(_, _) -> false. - join_types(Ts0, Ts1) -> if map_size(Ts0) < map_size(Ts1) -> @@ -1630,7 +1385,7 @@ join_types_1([V|Vs], Ts0, Ts1) -> {#{V:=Same},#{V:=Same}} -> join_types_1(Vs, Ts0, Ts1); {#{V:=T0},#{V:=T1}} -> - case join(T0, T1) of + case beam_types:join(T0, T1) of T1 -> join_types_1(Vs, Ts0, Ts1); T -> @@ -1642,326 +1397,21 @@ join_types_1([V|Vs], Ts0, Ts1) -> join_types_1([], Ts0, Ts1) -> maps:merge(Ts0, Ts1). -join([T1,T2|Ts]) -> - join([join(T1, T2)|Ts]); -join([T]) -> T. - -get_literal_from_type(#t_atom{elements=[Atom]}) -> - #b_literal{val=Atom}; -get_literal_from_type(#t_integer{elements={Int,Int}}) -> - #b_literal{val=Int}; -get_literal_from_type(nil) -> - #b_literal{val=[]}; -get_literal_from_type(_) -> none. - -remove_element_info(#t_integer{elements={Min,Max}}, Es) -> - foldl(fun(El, Acc) when Min =< El, El =< Max -> - maps:remove(El, Acc); - (_El, Acc) -> Acc - end, Es, maps:keys(Es)). - -t_atom() -> - #t_atom{elements=any}. - -t_atom(Atom) when is_atom(Atom) -> - #t_atom{elements=[Atom]}. - -t_boolean() -> - #t_atom{elements=[false,true]}. - -t_integer() -> - #t_integer{elements=any}. - -t_integer(Int) when is_integer(Int) -> - #t_integer{elements={Int,Int}}. - -t_integer(Min, Max) when is_integer(Min), is_integer(Max) -> - #t_integer{elements={Min,Max}}. - -t_is_boolean(#t_atom{elements=[F,T]}) -> - F =:= false andalso T =:= true; -t_is_boolean(#t_atom{elements=[B]}) -> - is_boolean(B); -t_is_boolean(_) -> false. - -t_tuple_size(#t_tuple{size=Size,exact=false}) -> - {at_least,Size}; -t_tuple_size(#t_tuple{size=Size,exact=true}) -> - {exact,Size}; -t_tuple_size(_) -> - none. - -is_singleton_type(Type) -> - get_literal_from_type(Type) =/= none. - -get_element_type(Index, Es) -> - case Es of - #{ Index := T } -> T; - #{} -> any - end. - -set_element_type(_Key, none, Es) -> - Es; -set_element_type(Key, any, Es) -> - maps:remove(Key, Es); -set_element_type(Key, Type, Es) -> - Es#{ Key => Type }. - -%% join(Type1, Type2) -> Type -%% Return the "join" of Type1 and Type2. The join is a more general -%% type than Type1 and Type2. For example: -%% -%% join(#t_integer{elements=any}, #t_integer=elements={0,3}}) -> -%% #t_integer{} -%% -%% The join for two different types result in 'any', which is -%% the top element for our type lattice: -%% -%% join(#t_integer{}, map) -> any - --spec join(type(), type()) -> type(). - -join(T, T) -> - verified_type(T); -join(none, T) -> - verified_type(T); -join(T, none) -> - verified_type(T); -join(any, _) -> any; -join(_, any) -> any; -join(#t_atom{elements=[_|_]=Set1}, #t_atom{elements=[_|_]=Set2}) -> - Set = ordsets:union(Set1, Set2), - case ordsets:size(Set) of - Size when Size =< ?ATOM_SET_SIZE -> - #t_atom{elements=Set}; - _Size -> - #t_atom{elements=any} - end; -join(#t_atom{elements=any}=T, #t_atom{elements=[_|_]}) -> T; -join(#t_atom{elements=[_|_]}, #t_atom{elements=any}=T) -> T; -join({binary,U1}, {binary,U2}) -> - {binary,gcd(U1, U2)}; -join(#t_integer{}, #t_integer{}) -> t_integer(); -join(list, cons) -> list; -join(cons, list) -> list; -join(nil, cons) -> list; -join(cons, nil) -> list; -join(nil, list) -> list; -join(list, nil) -> list; -join(#t_integer{}, float) -> number; -join(float, #t_integer{}) -> number; -join(#t_integer{}, number) -> number; -join(number, #t_integer{}) -> number; -join(float, number) -> number; -join(number, float) -> number; -join(#t_tuple{size=Sz,exact=ExactA,elements=EsA}, - #t_tuple{size=Sz,exact=ExactB,elements=EsB}) -> - Exact = ExactA and ExactB, - Es = join_tuple_elements(Sz, EsA, EsB), - #t_tuple{size=Sz,exact=Exact,elements=Es}; -join(#t_tuple{size=SzA,elements=EsA}, #t_tuple{size=SzB,elements=EsB}) -> - Sz = min(SzA, SzB), - Es = join_tuple_elements(Sz, EsA, EsB), - #t_tuple{size=Sz,elements=Es}; -join(_T1, _T2) -> - %%io:format("~p ~p\n", [_T1,_T2]), - any. - -join_tuple_elements(MinSize, EsA, EsB) -> - Es0 = join_elements(EsA, EsB), - maps:filter(fun(Index, _Type) -> Index =< MinSize end, Es0). - -join_elements(Es1, Es2) -> - Keys = if - map_size(Es1) =< map_size(Es2) -> maps:keys(Es1); - map_size(Es1) > map_size(Es2) -> maps:keys(Es2) - end, - join_elements_1(Keys, Es1, Es2, #{}). - -join_elements_1([Key | Keys], Es1, Es2, Acc0) -> - case {Es1, Es2} of - {#{ Key := Type1 }, #{ Key := Type2 }} -> - Acc = set_element_type(Key, join(Type1, Type2), Acc0), - join_elements_1(Keys, Es1, Es2, Acc); - {#{}, #{}} -> - join_elements_1(Keys, Es1, Es2, Acc0) - end; -join_elements_1([], _Es1, _Es2, Acc) -> - Acc. - -gcd(A, B) -> - case A rem B of - 0 -> B; - X -> gcd(B, X) - end. - meet_types([{V,T0}|Vs], Ts) -> #{V:=T1} = Ts, - case meet(T0, T1) of + case beam_types:meet(T0, T1) of + none -> none; T1 -> meet_types(Vs, Ts); T -> meet_types(Vs, Ts#{V:=T}) end; meet_types([], Ts) -> Ts. -meet([T1,T2|Ts]) -> - meet([meet(T1, T2)|Ts]); -meet([T]) -> T. - subtract_types([{V,T0}|Vs], Ts) -> #{V:=T1} = Ts, - case subtract(T1, T0) of + case beam_types:subtract(T1, T0) of + none -> none; T1 -> subtract_types(Vs, Ts); T -> subtract_types(Vs, Ts#{V:=T}) end; subtract_types([], Ts) -> Ts. -%% subtract(Type1, Type2) -> Type. -%% Subtract Type2 from Type1. Example: -%% -%% subtract(list, cons) -> nil - -subtract(#t_atom{elements=[_|_]=Set0}, #t_atom{elements=[_|_]=Set1}) -> - case ordsets:subtract(Set0, Set1) of - [] -> none; - [_|_]=Set -> #t_atom{elements=Set} - end; -subtract(number, float) -> #t_integer{}; -subtract(number, #t_integer{elements=any}) -> float; -subtract(list, cons) -> nil; -subtract(list, nil) -> cons; -subtract(T, _) -> T. - -%% meet(Type1, Type2) -> Type -%% Return the "meet" of Type1 and Type2. The meet is a narrower -%% type than Type1 and Type2. For example: -%% -%% meet(#t_integer{elements=any}, #t_integer{elements={0,3}}) -> -%% #t_integer{elements={0,3}} -%% -%% The meet for two different types result in 'none', which is -%% the bottom element for our type lattice: -%% -%% meet(#t_integer{}, map) -> none - --spec meet(type(), type()) -> type(). - -meet(T, T) -> - verified_type(T); -meet(#t_atom{elements=[_|_]=Set1}, #t_atom{elements=[_|_]=Set2}) -> - case ordsets:intersection(Set1, Set2) of - [] -> - none; - [_|_]=Set -> - #t_atom{elements=Set} - end; -meet(#t_atom{elements=[_|_]}=T, #t_atom{elements=any}) -> - T; -meet(#t_atom{elements=any}, #t_atom{elements=[_|_]}=T) -> - T; -meet(#t_integer{elements={_,_}}=T, #t_integer{elements=any}) -> - T; -meet(#t_integer{elements=any}, #t_integer{elements={_,_}}=T) -> - T; -meet(#t_integer{elements={Min1,Max1}}, - #t_integer{elements={Min2,Max2}}) -> - #t_integer{elements={max(Min1, Min2),min(Max1, Max2)}}; -meet(#t_integer{}=T, number) -> T; -meet(float=T, number) -> T; -meet(number, #t_integer{}=T) -> T; -meet(number, float=T) -> T; -meet(list, cons) -> cons; -meet(list, nil) -> nil; -meet(cons, list) -> cons; -meet(nil, list) -> nil; -meet(#t_tuple{}=T1, #t_tuple{}=T2) -> - meet_tuples(T1, T2); -meet({binary,U1}, {binary,U2}) -> - {binary,max(U1, U2)}; -meet(any, T) -> - verified_type(T); -meet(T, any) -> - verified_type(T); -meet(_, _) -> - %% Inconsistent types. There will be an exception at runtime. - none. - -meet_tuples(#t_tuple{size=Sz1,exact=true}, - #t_tuple{size=Sz2,exact=true}) when Sz1 =/= Sz2 -> - none; -meet_tuples(#t_tuple{size=Sz1,exact=Ex1,elements=Es1}, - #t_tuple{size=Sz2,exact=Ex2,elements=Es2}) -> - Size = max(Sz1, Sz2), - Exact = Ex1 or Ex2, - case meet_elements(Es1, Es2) of - none -> - none; - Es -> - #t_tuple{size=Size,exact=Exact,elements=Es} - end. - -meet_elements(Es1, Es2) -> - Keys = maps:keys(Es1) ++ maps:keys(Es2), - meet_elements_1(Keys, Es1, Es2, #{}). - -meet_elements_1([Key | Keys], Es1, Es2, Acc) -> - case {Es1, Es2} of - {#{ Key := Type1 }, #{ Key := Type2 }} -> - case meet(Type1, Type2) of - none -> none; - Type -> meet_elements_1(Keys, Es1, Es2, Acc#{ Key => Type }) - end; - {#{ Key := Type1 }, _} -> - meet_elements_1(Keys, Es1, Es2, Acc#{ Key => Type1 }); - {_, #{ Key := Type2 }} -> - meet_elements_1(Keys, Es1, Es2, Acc#{ Key => Type2 }) - end; -meet_elements_1([], _Es1, _Es2, Acc) -> - Acc. - -%% verified_type(Type) -> Type -%% Returns the passed in type if it is one of the defined types. -%% Crashes if there is anything wrong with the type. -%% -%% Here are all possible types: -%% -%% any Any Erlang term (top element for the type lattice). -%% -%% #t_atom{} Any atom or some specific atoms. -%% {binary,Unit} Binary/bitstring aligned to unit Unit. -%% float Floating point number. -%% #t_integer{} Integer -%% list Empty or nonempty list. -%% map Map. -%% nil Empty list. -%% cons Cons (nonempty list). -%% number A number (float or integer). -%% #t_tuple{} Tuple. -%% -%% none No type (bottom element for the type lattice). - --spec verified_type(T) -> T when - T :: type(). - -verified_type(any=T) -> T; -verified_type(none=T) -> T; -verified_type(#t_atom{elements=any}=T) -> T; -verified_type(#t_atom{elements=[_|_]}=T) -> T; -verified_type({binary,U}=T) when is_integer(U) -> T; -verified_type(#t_integer{elements=any}=T) -> T; -verified_type(#t_integer{elements={Min,Max}}=T) - when is_integer(Min), is_integer(Max) -> T; -verified_type(list=T) -> T; -verified_type(map=T) -> T; -verified_type(nil=T) -> T; -verified_type(cons=T) -> T; -verified_type(number=T) -> T; -verified_type(#t_tuple{size=Size,elements=Es}=T) -> - %% All known elements must have a valid index and type. 'any' is prohibited - %% since it's implicit and should never be present in the map. - maps:fold(fun(Index, Element, _) when is_integer(Index), - 1 =< Index, Index =< Size, - Element =/= any, Element =/= none -> - verified_type(Element) - end, [], Es), - T; -verified_type(float=T) -> T. diff --git a/lib/compiler/src/beam_trim.erl b/lib/compiler/src/beam_trim.erl index acf3838da4..ad8839cc7d 100644 --- a/lib/compiler/src/beam_trim.erl +++ b/lib/compiler/src/beam_trim.erl @@ -244,6 +244,9 @@ remap([{make_fun2,_,_,_,_}=I|T], Map, Acc) -> remap([{deallocate,N}|Is], Map, Acc) -> I = {deallocate,Map({frame_size,N})}, remap(Is, Map, [I|Acc]); +remap([{swap,Reg1,Reg2}|Is], Map, Acc) -> + I = {swap,Map(Reg1),Map(Reg2)}, + remap(Is, Map, [I|Acc]); remap([{test,Name,Fail,Ss}|Is], Map, Acc) -> I = {test,Name,Fail,[Map(S) || S <- Ss]}, remap(Is, Map, [I|Acc]); @@ -382,6 +385,8 @@ frame_size([{bs_set_position,_,_}|Is], Safe) -> frame_size(Is, Safe); frame_size([{bs_get_tail,_,_,_}|Is], Safe) -> frame_size(Is, Safe); +frame_size([{swap,_,_}|Is], Safe) -> + frame_size(Is, Safe); frame_size(_, _) -> throw(not_possible). frame_size_branch(0, Is, Safe) -> @@ -444,6 +449,8 @@ is_not_used(Y, [{line,_}|Is]) -> is_not_used(Y, Is); is_not_used(Y, [{make_fun2,_,_,_,_}|Is]) -> is_not_used(Y, Is); +is_not_used(Y, [{swap,Reg1,Reg2}|Is]) -> + Y =/= Reg1 andalso Y =/= Reg2 andalso is_not_used(Y, Is); is_not_used(Y, [{test,_,_,Ss}|Is]) -> not member(Y, Ss) andalso is_not_used(Y, Is); is_not_used(Y, [{test,_Op,{f,_},_Live,Ss,Dst}|Is]) -> diff --git a/lib/compiler/src/beam_types.erl b/lib/compiler/src/beam_types.erl new file mode 100644 index 0000000000..821ccd31bb --- /dev/null +++ b/lib/compiler/src/beam_types.erl @@ -0,0 +1,778 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(beam_types). + +-include("beam_types.hrl"). + +-import(lists, [foldl/3, reverse/1, reverse/2]). + +-export([meet/1, meet/2, join/1, join/2, subtract/2]). + +-export([get_singleton_value/1, + is_singleton_type/1, + is_boolean_type/1, + normalize/1]). + +-export([get_element_type/2, set_element_type/3]). + +-export([make_type_from_value/1]). + +-export([make_atom/1, + make_boolean/0, + make_integer/1, + make_integer/2]). + +-define(IS_LIST_TYPE(N), + N =:= list orelse + N =:= cons orelse + N =:= nil). + +-define(IS_NUMBER_TYPE(N), + N =:= number orelse + N =:= float orelse + is_record(N, t_integer)). + +-define(TUPLE_SET_LIMIT, 20). + +%% Folds meet/2 over a list. + +-spec meet([type()]) -> type(). + +meet([T1, T2 | Ts]) -> + meet([meet(T1, T2) | Ts]); +meet([T]) -> T. + +%% Return the "meet" of Type1 and Type2, which is more general than Type1 and +%% Type2. This is identical to glb/2 but can operate on and produce unions. +%% +%% A = #t_union{list=nil, number=[number], other=[#t_map{}]} +%% B = #t_union{number=[#t_integer{}], other=[#t_map{}]} +%% +%% meet(A, B) -> +%% #t_union{number=[#t_integer{}], other=[#t_map{}]} +%% +%% The meet of two different types result in 'none', which is the bottom +%% element for our type lattice: +%% +%% meet(#t_integer{}, #t_map{}) -> none + +-spec meet(type(), type()) -> type(). + +meet(T, T) -> + verified_type(T); +meet(any, T) -> + verified_type(T); +meet(T, any) -> + verified_type(T); +meet(#t_union{}=A, B) -> + meet_unions(A, B); +meet(A, #t_union{}=B) -> + meet_unions(B, A); +meet(A, B) -> + glb(A, B). + +meet_unions(#t_union{atom=AtomA,list=ListA,number=NumberA, + tuple_set=TSetA,other=OtherA}, + #t_union{atom=AtomB,list=ListB,number=NumberB, + tuple_set=TSetB,other=OtherB}) -> + Union = #t_union{atom=glb(AtomA, AtomB), + list=glb(ListA, ListB), + number=glb(NumberA, NumberB), + tuple_set=meet_tuple_sets(TSetA, TSetB), + other=glb(OtherA, OtherB)}, + shrink_union(Union); +meet_unions(#t_union{atom=AtomA}, #t_atom{}=B) -> + case glb(AtomA, B) of + none -> none; + Atom -> Atom + end; +meet_unions(#t_union{number=NumberA}, B) when ?IS_NUMBER_TYPE(B) -> + case glb(NumberA, B) of + none -> none; + Number -> Number + end; +meet_unions(#t_union{list=ListA}, B) when ?IS_LIST_TYPE(B) -> + case glb(ListA, B) of + none -> none; + List -> List + end; +meet_unions(#t_union{tuple_set=Tuples}, #t_tuple{}=B) -> + Set = meet_tuple_sets(Tuples, new_tuple_set(B)), + shrink_union(#t_union{tuple_set=Set}); +meet_unions(#t_union{other=OtherA}, OtherB) -> + case glb(OtherA, OtherB) of + none -> none; + Other -> Other + end. + +meet_tuple_sets(none, _) -> + none; +meet_tuple_sets(_, none) -> + none; +meet_tuple_sets(#t_tuple{}=A, #t_tuple{}=B) -> + new_tuple_set(glb(A, B)); +meet_tuple_sets(#t_tuple{}=Tuple, Records) -> + mts_tuple(Records, Tuple, []); +meet_tuple_sets(Records, #t_tuple{}=Tuple) -> + meet_tuple_sets(Tuple, Records); +meet_tuple_sets(RecordsA, RecordsB) -> + mts_records(RecordsA, RecordsB). + +mts_tuple([{Key, Type} | Records], Tuple, Acc) -> + case glb(Type, Tuple) of + none -> mts_tuple(Records, Tuple, Acc); + T -> mts_tuple(Records, Tuple, [{Key, T} | Acc]) + end; +mts_tuple([], _Tuple, [_|_]=Acc) -> + reverse(Acc); +mts_tuple([], _Tuple, []) -> + none. + +mts_records(RecordsA, RecordsB) -> + mts_records(RecordsA, RecordsB, []). + +mts_records([{Key, A} | RsA], [{Key, B} | RsB], Acc) -> + case glb(A, B) of + none -> mts_records(RsA, RsB, Acc); + T -> mts_records(RsA, RsB, [{Key, T} | Acc]) + end; +mts_records([{KeyA, _} | _ ]=RsA, [{KeyB, _} | RsB], Acc) when KeyA > KeyB -> + mts_records(RsA, RsB, Acc); +mts_records([{KeyA, _} | RsA], [{KeyB, _} | _] = RsB, Acc) when KeyA < KeyB -> + mts_records(RsA, RsB, Acc); +mts_records(_RsA, [], [_|_]=Acc) -> + reverse(Acc); +mts_records([], _RsB, [_|_]=Acc) -> + reverse(Acc); +mts_records(_RsA, _RsB, []) -> + none. + +%% Folds join/2 over a list. + +-spec join([type()]) -> type(). + +join([T1, T2| Ts]) -> + join([join(T1, T2) | Ts]); +join([T]) -> T. + +%% Return the "join" of Type1 and Type2, which is more general than Type1 and +%% Type2. This is identical to lub/2 but can operate on and produce unions. +%% +%% join(#t_integer{}, #t_map{}) -> #t_union{number=[#t_integer{}], +%% other=[#t_map{}]} + +-spec join(type(), type()) -> type(). + +join(T, T) -> T; +join(_T, any) -> any; +join(any, _T) -> any; +join(T, none) -> T; +join(none, T) -> T; + +join(#t_union{}=A, B) -> + join_unions(A, B); +join(A, #t_union{}=B) -> + join_unions(B, A); + +%% Union creation... +join(#t_atom{}=A, #t_atom{}=B) -> + lub(A, B); +join(#t_atom{}=A, B) when ?IS_LIST_TYPE(B) -> + #t_union{atom=A,list=B}; +join(#t_atom{}=A, B) when ?IS_NUMBER_TYPE(B) -> + #t_union{atom=A,number=B}; +join(#t_atom{}=A, #t_tuple{}=B) -> + #t_union{atom=A,tuple_set=new_tuple_set(B)}; +join(#t_atom{}=A, B) -> + #t_union{atom=A,other=B}; +join(A, #t_atom{}=B) -> + join(B, A); + +join(A, B) when ?IS_LIST_TYPE(A), ?IS_LIST_TYPE(B) -> + lub(A, B); +join(A, B) when ?IS_LIST_TYPE(A), ?IS_NUMBER_TYPE(B) -> + #t_union{list=A,number=B}; +join(A, #t_tuple{}=B) when ?IS_LIST_TYPE(A) -> + #t_union{list=A,tuple_set=new_tuple_set(B)}; +join(A, B) when ?IS_LIST_TYPE(A) -> + #t_union{list=A,other=B}; +join(A, B) when ?IS_LIST_TYPE(B) -> + join(B, A); + +join(A, B) when ?IS_NUMBER_TYPE(A), ?IS_NUMBER_TYPE(B) -> + lub(A, B); +join(A, #t_tuple{}=B) when ?IS_NUMBER_TYPE(A) -> + #t_union{number=A,tuple_set=new_tuple_set(B)}; +join(A, B) when ?IS_NUMBER_TYPE(A) -> + #t_union{number=A,other=B}; +join(A, B) when ?IS_NUMBER_TYPE(B) -> + join(B, A); + +join(#t_tuple{}=A, #t_tuple{}=B) -> + case {record_key(A), record_key(B)} of + {Same, Same} -> + lub(A, B); + {none, _Key} -> + lub(A, B); + {_Key, none} -> + lub(A, B); + {KeyA, KeyB} when KeyA < KeyB -> + #t_union{tuple_set=[{KeyA, A}, {KeyB, B}]}; + {KeyA, KeyB} when KeyA > KeyB -> + #t_union{tuple_set=[{KeyB, B}, {KeyA, A}]} + end; +join(#t_tuple{}=A, B) -> + %% All other combinations have been tried already, so B must be 'other' + #t_union{tuple_set=new_tuple_set(A),other=B}; +join(A, #t_tuple{}=B) -> + join(B, A); + +join(A, B) -> + lub(A, B). + +join_unions(#t_union{atom=AtomA,list=ListA,number=NumberA, + tuple_set=TSetA,other=OtherA}, + #t_union{atom=AtomB,list=ListB,number=NumberB, + tuple_set=TSetB,other=OtherB}) -> + Union = #t_union{atom=lub(AtomA, AtomB), + list=lub(ListA, ListB), + number=lub(NumberA, NumberB), + tuple_set=join_tuple_sets(TSetA, TSetB), + other=lub(OtherA, OtherB)}, + shrink_union(Union); +join_unions(#t_union{atom=AtomA}=A, #t_atom{}=B) -> + A#t_union{atom=lub(AtomA, B)}; +join_unions(#t_union{list=ListA}=A, B) when ?IS_LIST_TYPE(B) -> + A#t_union{list=lub(ListA, B)}; +join_unions(#t_union{number=NumberA}=A, B) when ?IS_NUMBER_TYPE(B) -> + A#t_union{number=lub(NumberA, B)}; +join_unions(#t_union{tuple_set=TSetA}=A, #t_tuple{}=B) -> + Set = join_tuple_sets(TSetA, new_tuple_set(B)), + shrink_union(A#t_union{tuple_set=Set}); +join_unions(#t_union{other=OtherA}=A, B) -> + case lub(OtherA, B) of + any -> any; + T -> A#t_union{other=T} + end. + +join_tuple_sets(A, none) -> + A; +join_tuple_sets(none, B) -> + B; +join_tuple_sets(#t_tuple{}=A, #t_tuple{}=B) -> + lub(A, B); +join_tuple_sets(#t_tuple{}=Tuple, Records) -> + jts_tuple(Records, Tuple); +join_tuple_sets(Records, #t_tuple{}=Tuple) -> + join_tuple_sets(Tuple, Records); +join_tuple_sets(RecordsA, RecordsB) -> + jts_records(RecordsA, RecordsB). + +jts_tuple([{_Key, Tuple} | Records], Acc) -> + jts_tuple(Records, lub(Tuple, Acc)); +jts_tuple([], Acc) -> + Acc. + +jts_records(RsA, RsB) -> + jts_records(RsA, RsB, 0, []). + +jts_records(RsA, RsB, N, Acc) when N > ?TUPLE_SET_LIMIT -> + A = normalize_tuple_set(RsA, none), + B = normalize_tuple_set(RsB, A), + #t_tuple{} = normalize_tuple_set(Acc, B); +jts_records([{Key, A} | RsA], [{Key, B} | RsB], N, Acc) -> + jts_records(RsA, RsB, N + 1, [{Key, lub(A, B)} | Acc]); +jts_records([{KeyA, _} | _]=RsA, [{KeyB, B} | RsB], N, Acc) when KeyA > KeyB -> + jts_records(RsA, RsB, N + 1, [{KeyB, B} | Acc]); +jts_records([{KeyA, A} | RsA], [{KeyB, _} | _] = RsB, N, Acc) when KeyA < KeyB -> + jts_records(RsA, RsB, N + 1, [{KeyA, A} | Acc]); +jts_records([], RsB, _N, Acc) -> + reverse(Acc, RsB); +jts_records(RsA, [], _N, Acc) -> + reverse(Acc, RsA). + +%% Subtract Type2 from Type1. Example: +%% subtract(list, cons) -> nil + +-spec subtract(type(), type()) -> type(). + +subtract(#t_atom{elements=[_|_]=Set0}, #t_atom{elements=[_|_]=Set1}) -> + case ordsets:subtract(Set0, Set1) of + [] -> none; + [_|_]=Set -> #t_atom{elements=Set} + end; +subtract(number, float) -> #t_integer{}; +subtract(number, #t_integer{elements=any}) -> float; +subtract(list, cons) -> nil; +subtract(list, nil) -> cons; + +subtract(#t_union{atom=Atom}=A, #t_atom{}=B)-> + shrink_union(A#t_union{atom=subtract(Atom, B)}); +subtract(#t_union{number=Number}=A, B) when ?IS_NUMBER_TYPE(B) -> + shrink_union(A#t_union{number=subtract(Number, B)}); +subtract(#t_union{list=List}=A, B) when ?IS_LIST_TYPE(B) -> + shrink_union(A#t_union{list=subtract(List, B)}); +subtract(#t_union{tuple_set=[_|_]=Records0}=A, #t_tuple{}=B) -> + %% Filter out all records that are strictly more specific than B. + NewSet = case [{Key, T} || {Key, T} <- Records0, meet(T, B) =/= T] of + [_|_]=Records -> Records; + [] -> none + end, + shrink_union(A#t_union{tuple_set=NewSet}); +subtract(#t_union{tuple_set=#t_tuple{}=Tuple}=A, #t_tuple{}=B) -> + %% Exclude Tuple if it's strictly more specific than B. + case meet(Tuple, B) of + Tuple -> shrink_union(A#t_union{tuple_set=none}); + _ -> A + end; + +subtract(T, _) -> T. + +%%% +%%% Type operators +%%% + +-spec get_singleton_value(Type) -> Result when + Type :: type(), + Result :: {ok, term()} | error. +get_singleton_value(#t_atom{elements=[Atom]}) -> + {ok, Atom}; +get_singleton_value(#t_integer{elements={Int,Int}}) -> + {ok, Int}; +get_singleton_value(nil) -> + {ok, []}; +get_singleton_value(_) -> + error. + +-spec is_boolean_type(type()) -> boolean(). +is_boolean_type(#t_atom{elements=[F,T]}) -> + F =:= false andalso T =:= true; +is_boolean_type(#t_atom{elements=[B]}) -> + is_boolean(B); +is_boolean_type(#t_union{}=T) -> + is_boolean_type(normalize(T)); +is_boolean_type(_) -> + false. + +-spec is_singleton_type(type()) -> boolean(). +is_singleton_type(Type) -> + get_singleton_value(Type) =/= error. + +-spec set_element_type(Key, Type, Elements) -> Elements when + Key :: term(), + Type :: type(), + Elements :: elements(). +set_element_type(_Key, none, Es) -> + Es; +set_element_type(Key, any, Es) -> + maps:remove(Key, Es); +set_element_type(Key, Type, Es) -> + Es#{ Key => Type }. + +-spec get_element_type(Key, Elements) -> type() when + Key :: term(), + Elements :: elements(). +get_element_type(Index, Es) -> + case Es of + #{ Index := T } -> T; + #{} -> any + end. + +-spec normalize(type()) -> normal_type(). +normalize(#t_union{atom=Atom,list=List,number=Number, + tuple_set=Tuples,other=Other}) -> + A = lub(Atom, List), + B = lub(A, Number), + C = lub(B, Other), + normalize_tuple_set(Tuples, C); +normalize(T) -> + verified_normal_type(T). + +normalize_tuple_set([{_, A} | Records], B) -> + normalize_tuple_set(Records, lub(A, B)); +normalize_tuple_set([], B) -> + B; +normalize_tuple_set(A, B) -> + lub(A, B). + +%%% +%%% Type constructors +%%% + +-spec make_type_from_value(term()) -> type(). +make_type_from_value(Value) -> + mtfv_1(Value). + +mtfv_1([]) -> nil; +mtfv_1([_|_]) -> cons; +mtfv_1(A) when is_atom(A) -> #t_atom{elements=[A]}; +mtfv_1(B) when is_binary(B) -> #t_bitstring{unit=8}; +mtfv_1(B) when is_bitstring(B) -> #t_bitstring{}; +mtfv_1(F) when is_float(F) -> float; +mtfv_1(F) when is_function(F) -> + {arity, Arity} = erlang:fun_info(F, arity), + #t_fun{arity=Arity}; +mtfv_1(I) when is_integer(I) -> make_integer(I); +mtfv_1(M) when is_map(M) -> #t_map{}; +mtfv_1(T) when is_tuple(T) -> + {Es,_} = foldl(fun(Val, {Es0, Index}) -> + Type = mtfv_1(Val), + Es = set_element_type(Index, Type, Es0), + {Es, Index + 1} + end, {#{}, 1}, tuple_to_list(T)), + #t_tuple{exact=true,size=tuple_size(T),elements=Es}; +mtfv_1(_Term) -> + any. + +-spec make_atom(atom()) -> type(). +make_atom(Atom) when is_atom(Atom) -> + #t_atom{elements=[Atom]}. + +-spec make_boolean() -> type(). +make_boolean() -> + #t_atom{elements=[false,true]}. + +-spec make_integer(integer()) -> type(). +make_integer(Int) when is_integer(Int) -> + make_integer(Int, Int). + +-spec make_integer(Min, Max) -> type() when + Min :: integer(), + Max :: integer(). +make_integer(Min, Max) when is_integer(Min), is_integer(Max), Min =< Max -> + #t_integer{elements={Min,Max}}. + +%%% +%%% Helpers +%%% + +%% Return the greatest lower bound of the types Type1 and Type2. The GLB is a +%% more specific type than Type1 and Type2, and is always a normal type. +%% +%% glb(#t_integer{elements=any}, #t_integer{elements={0,3}}) -> +%% #t_integer{elements={0,3}} +%% +%% The GLB of two different types result in 'none', which is the bottom +%% element for our type lattice: +%% +%% glb(#t_integer{}, #t_map{}) -> none + +-spec glb(normal_type(), normal_type()) -> normal_type(). + +glb(T, T) -> + verified_normal_type(T); +glb(any, T) -> + verified_normal_type(T); +glb(T, any) -> + verified_normal_type(T); +glb(#t_atom{elements=[_|_]=Set1}, #t_atom{elements=[_|_]=Set2}) -> + case ordsets:intersection(Set1, Set2) of + [] -> + none; + [_|_]=Set -> + #t_atom{elements=Set} + end; +glb(#t_atom{elements=[_|_]}=T, #t_atom{elements=any}) -> + T; +glb(#t_atom{elements=any}, #t_atom{elements=[_|_]}=T) -> + T; +glb(#t_bs_context{slots=SlotCountA,valid=ValidSlotsA}, + #t_bs_context{slots=SlotCountB,valid=ValidSlotsB}) -> + CommonSlotMask = (1 bsl min(SlotCountA, SlotCountB)) - 1, + CommonSlotsA = ValidSlotsA band CommonSlotMask, + CommonSlotsB = ValidSlotsB band CommonSlotMask, + if + CommonSlotsA =:= CommonSlotsB -> + #t_bs_context{slots=max(SlotCountA, SlotCountB), + valid=ValidSlotsA bor ValidSlotsB}; + CommonSlotsA =/= CommonSlotsB -> + none + end; +glb(#t_fun{arity=any}, #t_fun{}=T) -> + T; +glb(#t_fun{}=T, #t_fun{arity=any}) -> + T; +glb(#t_integer{elements={_,_}}=T, #t_integer{elements=any}) -> + T; +glb(#t_integer{elements=any}, #t_integer{elements={_,_}}=T) -> + T; +glb(#t_integer{elements={MinA,MaxA}}, #t_integer{elements={MinB,MaxB}}) + when MinA >= MinB, MinA =< MaxB; + MinB >= MinA, MinB =< MaxA -> + true = MinA =< MaxA andalso MinB =< MaxB, %Assertion. + #t_integer{elements={max(MinA, MinB),min(MaxA, MaxB)}}; +glb(#t_integer{}=T, number) -> T; +glb(float=T, number) -> T; +glb(number, #t_integer{}=T) -> T; +glb(number, float=T) -> T; +glb(list, cons) -> cons; +glb(list, nil) -> nil; +glb(cons, list) -> cons; +glb(nil, list) -> nil; +glb(#t_tuple{}=T1, #t_tuple{}=T2) -> + glb_tuples(T1, T2); +glb(#t_bitstring{unit=U1}, #t_bitstring{unit=U2}) -> + #t_bitstring{unit=U1 * U2 div gcd(U1, U2)}; +glb(_, _) -> + %% Inconsistent types. There will be an exception at runtime. + none. + +glb_tuples(#t_tuple{size=Sz1,exact=true}, + #t_tuple{size=Sz2,exact=true}) when Sz1 =/= Sz2 -> + none; +glb_tuples(#t_tuple{size=Sz1,exact=Ex1,elements=Es1}, + #t_tuple{size=Sz2,exact=Ex2,elements=Es2}) -> + Size = max(Sz1, Sz2), + Exact = Ex1 or Ex2, + case glb_elements(Es1, Es2) of + none -> + none; + Es -> + #t_tuple{size=Size,exact=Exact,elements=Es} + end. + +glb_elements(Es1, Es2) -> + Keys = maps:keys(Es1) ++ maps:keys(Es2), + glb_elements_1(Keys, Es1, Es2, #{}). + +glb_elements_1([Key | Keys], Es1, Es2, Acc) -> + case {Es1, Es2} of + {#{ Key := Type1 }, #{ Key := Type2 }} -> + %% Note the use of meet/2; elements don't need to be normal types. + case meet(Type1, Type2) of + none -> none; + Type -> glb_elements_1(Keys, Es1, Es2, Acc#{ Key => Type }) + end; + {#{ Key := Type1 }, _} -> + glb_elements_1(Keys, Es1, Es2, Acc#{ Key => Type1 }); + {_, #{ Key := Type2 }} -> + glb_elements_1(Keys, Es1, Es2, Acc#{ Key => Type2 }) + end; +glb_elements_1([], _Es1, _Es2, Acc) -> + Acc. + +%% Return the least upper bound of the types Type1 and Type2. The LUB is a more +%% general type than Type1 and Type2, and is always a normal type. +%% +%% For example: +%% +%% lub(#t_integer{elements=any}, #t_integer=elements={0,3}}) -> +%% #t_integer{} +%% +%% The LUB for two different types result in 'any' (not a union type!), which +%% is the top element for our type lattice: +%% +%% lub(#t_integer{}, #t_map{}) -> any + +-spec lub(normal_type(), normal_type()) -> normal_type(). + +lub(T, T) -> + verified_normal_type(T); +lub(none, T) -> + verified_normal_type(T); +lub(T, none) -> + verified_normal_type(T); +lub(any, _) -> + any; +lub(_, any) -> + any; +lub(#t_atom{elements=[_|_]=Set1}, #t_atom{elements=[_|_]=Set2}) -> + Set = ordsets:union(Set1, Set2), + case ordsets:size(Set) of + Size when Size =< ?ATOM_SET_SIZE -> + #t_atom{elements=Set}; + _Size -> + #t_atom{elements=any} + end; +lub(#t_atom{elements=any}=T, #t_atom{elements=[_|_]}) -> T; +lub(#t_atom{elements=[_|_]}, #t_atom{elements=any}=T) -> T; +lub(#t_bitstring{unit=U1}, #t_bitstring{unit=U2}) -> + #t_bitstring{unit=gcd(U1, U2)}; +lub(#t_fun{}, #t_fun{}) -> + #t_fun{}; +lub(#t_integer{elements={MinA,MaxA}}, + #t_integer{elements={MinB,MaxB}}) -> + #t_integer{elements={min(MinA,MinB),max(MaxA,MaxB)}}; +lub(#t_bs_context{slots=SlotsA,valid=ValidA}, + #t_bs_context{slots=SlotsB,valid=ValidB}) -> + #t_bs_context{slots=min(SlotsA, SlotsB), + valid=ValidA band ValidB}; +lub(#t_integer{}, #t_integer{}) -> #t_integer{}; +lub(list, cons) -> list; +lub(cons, list) -> list; +lub(nil, cons) -> list; +lub(cons, nil) -> list; +lub(nil, list) -> list; +lub(list, nil) -> list; +lub(#t_integer{}, float) -> number; +lub(float, #t_integer{}) -> number; +lub(#t_integer{}, number) -> number; +lub(number, #t_integer{}) -> number; +lub(float, number) -> number; +lub(number, float) -> number; +lub(#t_tuple{size=Sz,exact=ExactA,elements=EsA}, + #t_tuple{size=Sz,exact=ExactB,elements=EsB}) -> + Exact = ExactA and ExactB, + Es = lub_tuple_elements(Sz, EsA, EsB), + #t_tuple{size=Sz,exact=Exact,elements=Es}; +lub(#t_tuple{size=SzA,elements=EsA}, #t_tuple{size=SzB,elements=EsB}) -> + Sz = min(SzA, SzB), + Es = lub_tuple_elements(Sz, EsA, EsB), + #t_tuple{size=Sz,elements=Es}; +lub(_T1, _T2) -> + %%io:format("~p ~p\n", [_T1,_T2]), + any. + +lub_tuple_elements(MinSize, EsA, EsB) -> + Es0 = lub_elements(EsA, EsB), + maps:filter(fun(Index, _Type) -> Index =< MinSize end, Es0). + +lub_elements(Es1, Es2) -> + Keys = if + map_size(Es1) =< map_size(Es2) -> maps:keys(Es1); + map_size(Es1) > map_size(Es2) -> maps:keys(Es2) + end, + lub_elements_1(Keys, Es1, Es2, #{}). + +lub_elements_1([Key | Keys], Es1, Es2, Acc0) -> + case {Es1, Es2} of + {#{ Key := Type1 }, #{ Key := Type2 }} -> + %% Note the use of join/2; elements don't need to be normal types. + Acc = set_element_type(Key, join(Type1, Type2), Acc0), + lub_elements_1(Keys, Es1, Es2, Acc); + {#{}, #{}} -> + lub_elements_1(Keys, Es1, Es2, Acc0) + end; +lub_elements_1([], _Es1, _Es2, Acc) -> + Acc. + +%% + +gcd(A, B) -> + case A rem B of + 0 -> B; + X -> gcd(B, X) + end. + +%% + +record_key(#t_tuple{exact=true,size=Size,elements=#{ 1 := Tag }}) -> + case is_singleton_type(Tag) of + true -> {Size, Tag}; + false -> none + end; +record_key(_) -> + none. + +new_tuple_set(T) -> + case record_key(T) of + none -> T; + Key -> [{Key, T}] + end. + +%% + +shrink_union(#t_union{other=any}) -> + any; +shrink_union(#t_union{atom=Atom,list=none,number=none, + tuple_set=none,other=none}) -> + Atom; +shrink_union(#t_union{atom=none,list=List,number=none, + tuple_set=none,other=none}) -> + List; +shrink_union(#t_union{atom=none,list=none,number=Number, + tuple_set=none,other=none}) -> + Number; +shrink_union(#t_union{atom=none,list=none,number=none, + tuple_set=#t_tuple{}=Tuple,other=none}) -> + Tuple; +shrink_union(#t_union{atom=none,list=none,number=none, + tuple_set=[{_Key, Record}],other=none}) -> + #t_tuple{} = Record; %Assertion. +shrink_union(#t_union{atom=none,list=none,number=none, + tuple_set=none,other=Other}) -> + Other; +shrink_union(#t_union{}=T) -> + T. + +%% Verifies that the given type is well-formed. + +-spec verified_type(T) -> T when + T :: type(). + +verified_type(#t_union{atom=Atom, + list=List, + number=Number, + tuple_set=TSet, + other=Other}=T) -> + _ = verified_normal_type(Atom), + _ = verified_normal_type(List), + _ = verified_normal_type(Number), + _ = verify_tuple_set(TSet), + _ = verified_normal_type(Other), + T; +verified_type(T) -> + verified_normal_type(T). + +verify_tuple_set([_|_]=T) -> + _ = [verified_normal_type(Rec) || {_, Rec} <- T], + T; +verify_tuple_set(#t_tuple{}=T) -> + none = record_key(T), %Assertion. + T; +verify_tuple_set(none=T) -> + T. + +-spec verified_normal_type(T) -> T when + T :: normal_type(). + +verified_normal_type(any=T) -> T; +verified_normal_type(none=T) -> T; +verified_normal_type(#t_atom{elements=any}=T) -> T; +verified_normal_type(#t_atom{elements=[_|_]}=T) -> T; +verified_normal_type(#t_bitstring{unit=U}=T) + when is_integer(U), U >= 1 -> + T; +verified_normal_type(#t_bs_context{}=T) -> T; +verified_normal_type(#t_fun{arity=Arity}=T) + when Arity =:= any; is_integer(Arity) -> + T; +verified_normal_type(float=T) -> T; +verified_normal_type(#t_integer{elements=any}=T) -> T; +verified_normal_type(#t_integer{elements={Min,Max}}=T) + when is_integer(Min), is_integer(Max), Min =< Max -> + T; +verified_normal_type(list=T) -> T; +verified_normal_type(#t_map{}=T) -> T; +verified_normal_type(nil=T) -> T; +verified_normal_type(cons=T) -> T; +verified_normal_type(number=T) -> T; +verified_normal_type(#t_tuple{size=Size,elements=Es}=T) -> + %% All known elements must have a valid index and type (which may be a + %% union). 'any' is prohibited since it's implicit and should never be + %% present in the map, and a 'none' element ought to have reduced the + %% entire tuple to 'none'. + maps:fold(fun(Index, Element, _) when is_integer(Index), + 1 =< Index, Index =< Size, + Element =/= any, Element =/= none -> + verified_type(Element) + end, [], Es), + T. diff --git a/lib/compiler/src/beam_types.hrl b/lib/compiler/src/beam_types.hrl new file mode 100644 index 0000000000..09f87d61ba --- /dev/null +++ b/lib/compiler/src/beam_types.hrl @@ -0,0 +1,88 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% Common term types for passes operating on beam SSA and assembly. Helper +%% functions for wrangling these can be found in beam_types.erl +%% +%% The type lattice is as follows: +%% +%% any Any Erlang term (top element). +%% +%% - #t_atom{} Atom, or a set thereof. +%% - #t_bitstring{} Bitstring. +%% - #t_bs_context{} Match context. +%% - #t_fun{} Fun. +%% - #t_map{} Map. +%% - number Any number. +%% -- float Floating point number. +%% -- integer Integer. +%% - list Any list. +%% -- cons Cons (nonempty list). +%% -- nil The empty list. +%% - #t_tuple{} Tuple. +%% +%% none No type (bottom element). +%% +%% We also use #t_union{} to represent conflicting types produced by certain +%% expressions, e.g. the "#t_atom{} or #t_tuple{}" of lists:keyfind/3, which is +%% very useful for preserving type information when we would otherwise have +%% reduced it to 'any'. Since few operations can make direct use of this extra +%% type information, types should generally be normalized to one of the above +%% before use. + +-define(ATOM_SET_SIZE, 5). + +-record(t_atom, {elements=any :: 'any' | [atom()]}). +-record(t_fun, {arity=any :: arity() | 'any'}). +-record(t_integer, {elements=any :: 'any' | {integer(),integer()}}). +-record(t_bitstring, {unit=1 :: pos_integer()}). +-record(t_bs_context, {slots=0 :: non_neg_integer(), + valid=0 :: non_neg_integer()}). +-record(t_map, {elements=#{} :: map_elements()}). +-record(t_tuple, {size=0 :: integer(), + exact=false :: boolean(), + elements=#{} :: tuple_elements()}). + +%% Known element types, unknown elements are assumed to be 'any'. The key is +%% a 1-based integer index for tuples, and a plain literal for maps (that is, +%% not wrapped in a #b_literal{}, just the value itself). + +-type tuple_elements() :: #{ Key :: pos_integer() => type() }. +-type map_elements() :: #{ Key :: term() => type() }. + +-type elements() :: tuple_elements() | map_elements(). + +-type normal_type() :: any | none | + list | number | + #t_atom{} | #t_bitstring{} | #t_bs_context{} | + #t_fun{} | #t_integer{} | #t_map{} | #t_tuple{} | + 'cons' | 'float' | 'nil'. + +-type record_key() :: {Arity :: integer(), Tag :: normal_type() }. +-type record_set() :: ordsets:ordset({record_key(), #t_tuple{}}). +-type tuple_set() :: #t_tuple{} | record_set(). + +-record(t_union, {atom=none :: none | #t_atom{}, + list=none :: none | list | cons | nil, + number=none :: none | number | float | #t_integer{}, + tuple_set=none :: none | tuple_set(), + other=none :: normal_type()}). + +-type type() :: #t_union{} | normal_type(). diff --git a/lib/compiler/src/beam_validator.erl b/lib/compiler/src/beam_validator.erl index 349d74eb58..d55aeed9a9 100644 --- a/lib/compiler/src/beam_validator.erl +++ b/lib/compiler/src/beam_validator.erl @@ -19,6 +19,10 @@ -module(beam_validator). +-include("beam_types.hrl"). + +-define(UNICODE_MAX, (16#10FFFF)). + -compile({no_auto_import,[min/2]}). %% Avoid warning for local function error/1 clashing with autoimported BIF. @@ -26,7 +30,6 @@ %% Interface for compiler. -export([module/2, format_error/1]). --export([type_anno/1, type_anno/2, type_anno/4]). -import(lists, [dropwhile/2,foldl/3,member/2,reverse/1,sort/1,zip/2]). @@ -45,34 +48,6 @@ module({Mod,Exp,Attr,Fs,Lc}=Code, _Opts) {error,[{atom_to_list(Mod),Es}]} end. -%% Provides a stable interface for type annotations, used by certain passes to -%% indicate that we can safely assume that a register has a given type. --spec type_anno(term()) -> term(). -type_anno(atom) -> {atom,[]}; -type_anno(bool) -> bool; -type_anno({binary,_}) -> binary; -type_anno(cons) -> cons; -type_anno(float) -> {float,[]}; -type_anno(integer) -> {integer,[]}; -type_anno(list) -> list; -type_anno(map) -> map; -type_anno(match_context) -> match_context; -type_anno(number) -> number; -type_anno(nil) -> nil. - --spec type_anno(term(), term()) -> term(). -type_anno(atom, Value) when is_atom(Value) -> {atom, Value}; -type_anno(float, Value) when is_float(Value) -> {float, Value}; -type_anno(integer, Value) when is_integer(Value) -> {integer, Value}. - --spec type_anno(term(), term(), term(), term()) -> term(). -type_anno(tuple, Size, Exact, Elements) when is_integer(Size), Size >= 0, - is_map(Elements) -> - case Exact of - true -> {tuple, Size, Elements}; - false -> {tuple, [Size], Elements} - end. - -spec format_error(term()) -> iolist(). format_error({{_M,F,A},{I,Off,limit}}) -> @@ -119,7 +94,7 @@ format_error(Error) -> %% format as used in the compiler and in .S files. validate(Module, Fs) -> - Ft = index_parameter_types(Fs, []), + Ft = build_function_table(Fs, []), validate_0(Module, Fs, Ft). validate_0(_Module, [], _) -> []; @@ -136,8 +111,15 @@ validate_0(Module, [{function,Name,Ar,Entry,Code}|Fs], Ft) -> erlang:raise(Class, Error, Stack) end. +-record(t_abstract, {kind}). + +%% The types are the same as in 'beam_types.hrl', with the addition of +%% #t_abstract{} that describes tuples under construction, match context +%% positions, and so on. +-type validator_type() :: #t_abstract{} | type(). + -record(value_ref, {id :: index()}). --record(value, {op :: term(), args :: [argument()], type :: type()}). +-record(value, {op :: term(), args :: [argument()], type :: validator_type()}). -type argument() :: #value_ref{} | literal(). @@ -149,34 +131,28 @@ validate_0(Module, [{function,Name,Ar,Entry,Code}|Fs], Ft) -> {literal, term()} | nil. --type tuple_sz() :: [non_neg_integer()] | %% Inexact - non_neg_integer(). %% Exact. - -%% Match context type. --record(ms, - {id=make_ref() :: reference(), %Unique ID. - valid=0 :: non_neg_integer(), %Valid slots - slots=0 :: non_neg_integer() %Number of slots - }). - --type type() :: binary | - cons | - list | - map | - nil | - #ms{} | - ms_position | - none | - number | - term | - tuple_in_progress | - {tuple, tuple_sz(), #{ literal() => type() }} | - literal(). - +%% Register tags describe the state of the register rather than the value they +%% contain (if any). +%% +%% initialized The register has been initialized with some valid term +%% so that it is safe to pass to the garbage collector. +%% NOT safe to use in any other way (will not crash the +%% emulator, but clearly points to a bug in the compiler). +%% +%% uninitialized The register contains any old garbage and can not be +%% passed to the garbage collector. +%% +%% {catchtag,[Lbl]} A special term used within a catch. Must only be used +%% by the catch instructions; NOT safe to use in other +%% instructions. +%% +%% {trytag,[Lbl]} A special term used within a try block. Must only be +%% used by the catch instructions; NOT safe to use in other +%% instructions. -type tag() :: initialized | uninitialized | - {catchtag, [label()]} | - {trytag, [label()]}. + {catchtag, ordsets:ordset(label())} | + {trytag, ordsets:ordset(label())}. -type x_regs() :: #{ {x, index()} => #value_ref{} }. -type y_regs() :: #{ {y, index()} => tag() | #value_ref{} }. @@ -200,11 +176,11 @@ validate_0(Module, [{function,Name,Ar,Entry,Code}|Fs], Ft) -> numy=none :: none | undecided | index(), %% Available heap size. h=0, - %Available heap size for floats. + %%Available heap size for floats. hf=0, %% Floating point state. fls=undefined, - %% List of hot catch/try labels + %% List of hot catch/try tags ct=[], %% Previous instruction was setelement/3. setelem=false, @@ -225,36 +201,32 @@ validate_0(Module, [{function,Name,Ar,Entry,Code}|Fs], Ft) -> branched=gb_trees:empty() :: branched_tab(), %% All defined labels labels=gb_sets:empty() :: label_set(), - %% Argument information of other functions in the module + %% Information of other functions in the module ft=gb_trees:empty() :: ft_tab(), %% Counter for #value_ref{} creation ref_ctr=0 :: index() }). -index_parameter_types([{function,_,_,Entry,Code0}|Fs], Acc0) -> +build_function_table([{function,_,Arity,Entry,Code0}|Fs], Acc0) -> Code = dropwhile(fun({label,L}) when L =:= Entry -> false; (_) -> true end, Code0), case Code of [{label,Entry}|Is] -> - Acc = index_parameter_types_1(Is, Entry, Acc0), - index_parameter_types(Fs, Acc); + Info = #{ arity => Arity, + parameter_types => find_parameter_types(Is, #{}) }, + build_function_table(Fs, [{Entry, Info} | Acc0]); _ -> - %% Something serious is wrong. Ignore it for now. + %% Something is seriously wrong. Ignore it for now. %% It will be detected and diagnosed later. - index_parameter_types(Fs, Acc0) + build_function_table(Fs, Acc0) end; -index_parameter_types([], Acc) -> +build_function_table([], Acc) -> gb_trees:from_orddict(sort(Acc)). -index_parameter_types_1([{'%', {type_info, Reg, Type0}} | Is], Entry, Acc) -> - Type = case Type0 of - match_context -> #ms{}; - _ -> Type0 - end, - Key = {Entry, Reg}, - index_parameter_types_1(Is, Entry, [{Key, Type} | Acc]); -index_parameter_types_1(_, _, Acc) -> +find_parameter_types([{'%', {type_info, Reg, Type}} | Is], Acc) -> + find_parameter_types(Is, Acc#{ Reg => Type }); +find_parameter_types(_, Acc) -> Acc. validate_1(Is, Name, Arity, Entry, Ft) -> @@ -326,7 +298,7 @@ init_vst(Arity, Ls1, Ls2, Ft) -> init_function_args(-1, Vst) -> Vst; init_function_args(X, Vst) -> - init_function_args(X - 1, create_term(term, argument, [], {x,X}, Vst)). + init_function_args(X - 1, create_term(any, argument, [], {x,X}, Vst)). kill_heap_allocation(St) -> St#st{h=0,hf=0}. @@ -381,17 +353,34 @@ valfun_1({try_case_end,Src}, Vst) -> kill_state(Vst); %% Instructions that cannot cause exceptions valfun_1({bs_get_tail,Ctx,Dst,Live}, Vst0) -> - bsm_validate_context(Ctx, Vst0), + assert_type(#t_bs_context{}, Ctx, Vst0), verify_live(Live, Vst0), verify_y_init(Vst0), Vst = prune_x_regs(Live, Vst0), - extract_term(binary, bs_get_tail, [Ctx], Dst, Vst, Vst0); + extract_term(#t_bitstring{}, bs_get_tail, [Ctx], Dst, Vst, Vst0); valfun_1(bs_init_writable=I, Vst) -> call(I, 1, Vst); valfun_1(build_stacktrace=I, Vst) -> call(I, 1, Vst); valfun_1({move,Src,Dst}, Vst) -> assign(Src, Dst, Vst); +valfun_1({swap,RegA,RegB}, Vst0) -> + assert_movable(RegA, Vst0), + assert_movable(RegB, Vst0), + + %% We don't expect fragile registers to be swapped. + %% Therefore, we can conservatively make both registers + %% fragile if one of the register is fragile instead of + %% swapping the fragility of the registers. + Sources = [RegA,RegB], + Vst1 = propagate_fragility(RegA, Sources, Vst0), + Vst2 = propagate_fragility(RegB, Sources, Vst1), + + %% Swap the value references. + VrefA = get_reg_vref(RegA, Vst2), + VrefB = get_reg_vref(RegB, Vst2), + Vst = set_reg_vref(VrefB, RegA, Vst2), + set_reg_vref(VrefA, RegB, Vst); valfun_1({fmove,Src,{fr,_}=Dst}, Vst) -> assert_type(float, Src, Vst), set_freg(Dst, Vst); @@ -399,25 +388,41 @@ valfun_1({fmove,{fr,_}=Src,Dst}, Vst0) -> assert_freg_set(Src, Vst0), assert_fls(checked, Vst0), Vst = eat_heap_float(Vst0), - create_term({float,[]}, fmove, [], Dst, Vst); + create_term(float, fmove, [], Dst, Vst); valfun_1({kill,Reg}, Vst) -> create_tag(initialized, kill, [], Reg, Vst); valfun_1({init,Reg}, Vst) -> create_tag(initialized, init, [], Reg, Vst); valfun_1({test_heap,Heap,Live}, Vst) -> test_heap(Heap, Live, Vst); -valfun_1({bif,Op,{f,_},Ss,Dst}=I, Vst) -> - case is_bif_safe(Op, length(Ss)) of - false -> - %% Since the BIF can fail, make sure that any catch state - %% is updated. - valfun_2(I, Vst); - true -> - %% It can't fail, so we finish handling it here (not updating - %% catch state). - validate_src(Ss, Vst), - Type = bif_return_type(Op, Ss, Vst), - extract_term(Type, {bif,Op}, Ss, Dst, Vst) +valfun_1({bif,Op,{f,0},Ss,Dst}=I, Vst0) -> + case will_bif_succeed(Op, Ss, Vst0) of + yes -> + %% This BIF cannot fail, handle it here without updating catch + %% state. + validate_bif(Op, cannot_fail, Ss, Dst, Vst0); + no -> + %% The stack will be scanned, so Y registers must be initialized. + Vst = branch_exception(Vst0), + verify_y_init(Vst), + kill_state(Vst); + maybe -> + %% The BIF can fail, make sure that any catch state is updated. + Vst = branch_exception(Vst0), + valfun_2(I, Vst) + end; +valfun_1({gc_bif,Op,{f,0},Live,Ss,Dst}=I, Vst0) -> + case will_bif_succeed(Op, Ss, Vst0) of + yes -> + validate_gc_bif(Op, cannot_fail, Ss, Dst, Live, Vst0); + no -> + Vst = branch_exception(Vst0), + verify_y_init(Vst), + kill_state(Vst); + maybe -> + Vst = branch_exception(Vst0), + assert_float_checked(Vst), + valfun_2(I, Vst) end; %% Put instructions. valfun_1({put_list,A,B,Dst}, Vst0) -> @@ -431,14 +436,15 @@ valfun_1({put_tuple2,Dst,{list,Elements}}, Vst0) -> Vst = eat_heap(Size+1, Vst0), {Es,_} = foldl(fun(Val, {Es0, Index}) -> Type = get_term_type(Val, Vst0), - Es = set_element_type({integer,Index}, Type, Es0), + Es = beam_types:set_element_type(Index, Type, Es0), {Es, Index + 1} end, {#{}, 1}, Elements), - Type = {tuple,Size,Es}, + Type = #t_tuple{exact=true,size=Size,elements=Es}, create_term(Type, put_tuple2, [], Dst, Vst); valfun_1({put_tuple,Sz,Dst}, Vst0) when is_integer(Sz) -> Vst1 = eat_heap(1, Vst0), - Vst = create_term(tuple_in_progress, put_tuple, [], Dst, Vst1), + Vst = create_term(#t_abstract{kind=unfinished_tuple}, put_tuple, [], + Dst, Vst1), #vst{current=St0} = Vst, St = St0#st{puts_left={Sz,{Dst,Sz,#{}}}}, Vst#vst{current=St}; @@ -450,15 +456,31 @@ valfun_1({put,Src}, Vst0) -> #st{puts_left=none} -> error(not_building_a_tuple); #st{puts_left={1,{Dst,Sz,Es0}}} -> - Es = Es0#{ {integer,Sz} => get_term_type(Src, Vst0) }, + ElementType = get_term_type(Src, Vst0), + Es = beam_types:set_element_type(Sz, ElementType, Es0), St = St0#st{puts_left=none}, - create_term({tuple,Sz,Es}, put_tuple, [], Dst, Vst#vst{current=St}); + Type = #t_tuple{exact=true,size=Sz,elements=Es}, + create_term(Type, put_tuple, [], Dst, Vst#vst{current=St}); #st{puts_left={PutsLeft,{Dst,Sz,Es0}}} when is_integer(PutsLeft) -> Index = Sz - PutsLeft + 1, - Es = Es0#{ {integer,Index} => get_term_type(Src, Vst0) }, + ElementType = get_term_type(Src, Vst0), + Es = beam_types:set_element_type(Index, ElementType, Es0), St = St0#st{puts_left={PutsLeft-1,{Dst,Sz,Es}}}, Vst#vst{current=St} end; +%% This instruction never fails, though it may be invalid in some contexts; see +%% val_dsetel/2 +valfun_1({set_tuple_element,Src,Tuple,N}, Vst) -> + I = N + 1, + assert_term(Src, Vst), + assert_type(#t_tuple{size=I}, Tuple, Vst), + %% Manually update the tuple type; we can't rely on the ordinary update + %% helpers as we must support overwriting (rather than just widening or + %% narrowing) known elements, and we can't use extract_term either since + %% the source tuple may be aliased. + #t_tuple{elements=Es0}=Type = normalize(get_term_type(Tuple, Vst)), + Es = beam_types:set_element_type(I, get_term_type(Src, Vst), Es0), + override_type(Type#t_tuple{elements=Es}, Tuple, Vst); %% Instructions for optimization of selective receives. valfun_1({recv_mark,{f,Fail}}, Vst) when is_integer(Fail) -> Vst; @@ -469,8 +491,13 @@ valfun_1(remove_message, Vst) -> %% The message term is no longer fragile. It can be used %% without restrictions. remove_fragility(Vst); -valfun_1({'%', {type_info, Reg, match_context}}, Vst) -> - update_type(fun meet/2, #ms{}, Reg, Vst); +valfun_1({'%', {type_info, _Reg, none}}, Vst) -> + %% Unreachable code, typically after a call that never returns. + kill_state(Vst); +valfun_1({'%', {type_info, Reg, #t_bs_context{}=Type}}, Vst) -> + %% This is a gross hack, but we'll be rid of it once we have proper union + %% types. + override_type(Type, Reg, Vst); valfun_1({'%', {type_info, Reg, Type}}, Vst) -> %% Explicit type information inserted by optimization passes to indicate %% that Reg has a certain type, so that we can accept cross-function type @@ -488,22 +515,42 @@ valfun_1({'%',_}, Vst) -> Vst; valfun_1({line,_}, Vst) -> Vst; -%% Exception generating calls -valfun_1({call_ext,Live,Func}=I, Vst) -> - case call_return_type(Func, Vst) of - exception -> - verify_live(Live, Vst), - %% The stack will be scanned, so Y registers - %% must be initialized. - verify_y_init(Vst), - kill_state(Vst); - _ -> - valfun_2(I, Vst) - end; +%% +%% Calls; these may be okay when the try/catch state or stack is undecided, +%% depending on whether they always succeed or always fail. +%% +valfun_1({apply,Live}, Vst) -> + validate_body_call(apply, Live+2, Vst); +valfun_1({apply_last,Live,N}, Vst) -> + validate_tail_call(N, apply, Live+2, Vst); +valfun_1({call_fun,Live}, Vst) -> + Fun = {x,Live}, + assert_term(Fun, Vst), + + %% An exception is raised on error, hence branching to 0. + branch(0, Vst, + fun(SuccVst0) -> + SuccVst = update_type(fun meet/2, #t_fun{arity=Live}, + Fun, SuccVst0), + validate_body_call('fun', Live+1, SuccVst) + end); +valfun_1({call,Live,Func}, Vst) -> + validate_body_call(Func, Live, Vst); +valfun_1({call_ext,Live,Func}, Vst) -> + validate_body_call(Func, Live, Vst); +valfun_1({call_only,Live,Func}, Vst) -> + validate_tail_call(none, Func, Live, Vst); +valfun_1({call_ext_only,Live,Func}, Vst) -> + validate_tail_call(none, Func, Live, Vst); +valfun_1({call_last,Live,Func,N}, Vst) -> + validate_tail_call(N, Func, Live, Vst); +valfun_1({call_ext_last,Live,Func,N}, Vst) -> + validate_tail_call(N, Func, Live, Vst); valfun_1(_I, #vst{current=#st{ct=undecided}}) -> error(unknown_catch_try_state); %% %% Allocate and deallocate, et.al +%% valfun_1({allocate,Stk,Live}, Vst) -> allocate(uninitialized, Stk, 0, Live, Vst); valfun_1({allocate_heap,Stk,Heap,Live}, Vst) -> @@ -530,103 +577,162 @@ valfun_1({'catch',Dst,{f,Fail}}, Vst) when Fail =/= none -> init_try_catch_branch(catchtag, Dst, Fail, Vst); valfun_1({'try',Dst,{f,Fail}}, Vst) when Fail =/= none -> init_try_catch_branch(trytag, Dst, Fail, Vst); -valfun_1({catch_end,Reg}, #vst{current=#st{ct=[Fail|_]}}=Vst0) -> +valfun_1({catch_end,Reg}, #vst{current=#st{ct=[Tag|_]}}=Vst0) -> case get_tag_type(Reg, Vst0) of - {catchtag,Fail} -> + {catchtag,_Fail}=Tag -> %% {x,0} contains the caught term, if any. - create_term(term, catch_end, [], {x,0}, kill_catch_tag(Reg, Vst0)); + create_term(any, catch_end, [], {x,0}, kill_catch_tag(Reg, Vst0)); Type -> error({wrong_tag_type,Type}) end; -valfun_1({try_end,Reg}, #vst{current=#st{ct=[Fail|_]}}=Vst) -> +valfun_1({try_end,Reg}, #vst{current=#st{ct=[Tag|_]}}=Vst) -> case get_tag_type(Reg, Vst) of - {trytag,Fail} -> + {trytag,_Fail}=Tag -> %% Kill the catch tag, note that x registers are unaffected. kill_catch_tag(Reg, Vst); Type -> error({wrong_tag_type,Type}) end; -valfun_1({try_case,Reg}, #vst{current=#st{ct=[Fail|_]}}=Vst0) -> +valfun_1({try_case,Reg}, #vst{current=#st{ct=[Tag|_]}}=Vst0) -> case get_tag_type(Reg, Vst0) of - {trytag,Fail} -> + {trytag,_Fail}=Tag -> %% Kill the catch tag and all x registers. Vst1 = prune_x_regs(0, kill_catch_tag(Reg, Vst0)), %% Class:Error:Stacktrace - Vst2 = create_term({atom,[]}, try_case, [], {x,0}, Vst1), - Vst = create_term(term, try_case, [], {x,1}, Vst2), - create_term(term, try_case, [], {x,2}, Vst); + Vst2 = create_term(#t_atom{}, try_case, [], {x,0}, Vst1), + Vst = create_term(any, try_case, [], {x,1}, Vst2), + create_term(any, try_case, [], {x,2}, Vst); Type -> error({wrong_tag_type,Type}) end; +%% Simple getters that can't fail. valfun_1({get_list,Src,D1,D2}, Vst0) -> assert_not_literal(Src), assert_type(cons, Src, Vst0), - Vst = extract_term(term, get_hd, [Src], D1, Vst0), - extract_term(term, get_tl, [Src], D2, Vst); + Vst = extract_term(any, get_hd, [Src], D1, Vst0), + extract_term(any, get_tl, [Src], D2, Vst); valfun_1({get_hd,Src,Dst}, Vst) -> assert_not_literal(Src), assert_type(cons, Src, Vst), - extract_term(term, get_hd, [Src], Dst, Vst); + extract_term(any, get_hd, [Src], Dst, Vst); valfun_1({get_tl,Src,Dst}, Vst) -> assert_not_literal(Src), assert_type(cons, Src, Vst), - extract_term(term, get_tl, [Src], Dst, Vst); + extract_term(any, get_tl, [Src], Dst, Vst); valfun_1({get_tuple_element,Src,N,Dst}, Vst) -> + Index = N+1, assert_not_literal(Src), - assert_type({tuple_element,N+1}, Src, Vst), - Index = {integer,N+1}, - Type = get_element_type(Index, Src, Vst), - extract_term(Type, {bif,element}, [Index, Src], Dst, Vst); + assert_type(#t_tuple{size=Index}, Src, Vst), + #t_tuple{elements=Es} = normalize(get_term_type(Src, Vst)), + Type = beam_types:get_element_type(Index, Es), + extract_term(Type, {bif,element}, [{integer,Index}, Src], Dst, Vst); valfun_1({jump,{f,Lbl}}, Vst) -> branch(Lbl, Vst, fun(SuccVst) -> %% The next instruction is never executed. kill_state(SuccVst) end); -valfun_1(I, Vst) -> +valfun_1(I, Vst0) -> + Vst = branch_exception(Vst0), valfun_2(I, Vst). -init_try_catch_branch(Tag, Dst, Fail, Vst0) -> - Vst1 = create_tag({Tag,[Fail]}, 'try_catch', [], Dst, Vst0), - #vst{current=#st{ct=Fails}=St0} = Vst1, - St = St0#st{ct=[[Fail]|Fails]}, - Vst = Vst0#vst{current=St}, +validate_tail_call(Deallocate, Func, Live, #vst{current=#st{numy=NumY}}=Vst0) -> + assert_float_checked(Vst0), + case will_call_succeed(Func, Vst0) of + yes when Deallocate =:= NumY -> + %% This call cannot fail, handle it without updating catch state. + tail_call(Func, Live, Vst0); + maybe when Deallocate =:= NumY -> + %% The call can fail, make sure that any catch state is updated. + Vst = branch_exception(Vst0), + tail_call(Func, Live, Vst); + no -> + %% The stack will be scanned, so Y registers must be initialized. + %% + %% Note that the compiler is allowed to emit garbage values for + %% "Deallocate" as we know that it will not be used in this case. + Vst = branch_exception(Vst0), + verify_live(Live, Vst), + verify_y_init(Vst), + kill_state(Vst); + _ when Deallocate =/= NumY -> + error({allocated, NumY}) + end. + +validate_body_call(Func, Live, + #vst{current=#st{numy=NumY}}=Vst0) when is_integer(NumY)-> + assert_float_checked(Vst0), + case will_call_succeed(Func, Vst0) of + yes -> + call(Func, Live, Vst0); + maybe -> + Vst = branch_exception(Vst0), + call(Func, Live, Vst); + no -> + Vst = branch_exception(Vst0), + verify_live(Live, Vst), + verify_y_init(Vst), + kill_state(Vst) + end; +validate_body_call(_, _, #vst{current=#st{numy=NumY}}) -> + error({allocated, NumY}). + +assert_float_checked(Vst) -> + case get_fls(Vst) of + undefined -> ok; + checked -> ok; + Fls -> error({unsafe_instruction,{float_error_state,Fls}}) + end. + +init_try_catch_branch(Kind, Dst, Fail, Vst0) -> + Tag = {Kind, [Fail]}, + Vst = create_tag(Tag, 'try_catch', [], Dst, Vst0), branch(Fail, Vst, - fun(CatchVst) -> - #vst{current=#st{ys=Ys}} = CatchVst, - maps:fold(fun init_catch_handler_1/3, CatchVst, Ys) + fun(CatchVst0) -> + %% We add the tag here because branch/4 rejects jumps to + %% labels referenced by try tags. + #vst{current=#st{ct=Tags,ys=Ys}=St0} = CatchVst0, + St = St0#st{ct=[Tag|Tags]}, + CatchVst = CatchVst0#vst{current=St}, + + maps:fold(fun init_catch_handler_1/3, CatchVst, Ys) end, - fun(SuccVst) -> - %% All potentially-throwing instructions after this - %% one will implicitly branch to the fail label; - %% see valfun_2/2 - SuccVst + fun(SuccVst0) -> + #vst{current=#st{ct=Tags}=St0} = SuccVst0, + St = St0#st{ct=[Tag|Tags]}, + SuccVst = SuccVst0#vst{current=St}, + + %% All potentially-throwing instructions after this one will + %% implicitly branch to the current try/catch handler; see + %% the base case of valfun_1/2 + SuccVst end). %% Set the initial state at the try/catch label. Assume that Y registers %% contain terms or try/catch tags. init_catch_handler_1(Reg, initialized, Vst) -> - create_term(term, 'catch_handler', [], Reg, Vst); + create_term(any, 'catch_handler', [], Reg, Vst); init_catch_handler_1(Reg, uninitialized, Vst) -> - create_term(term, 'catch_handler', [], Reg, Vst); + create_term(any, 'catch_handler', [], Reg, Vst); init_catch_handler_1(_, _, Vst) -> Vst. -valfun_2(I, #vst{current=#st{ct=[[Fail]|_]}}=Vst) when is_integer(Fail) -> +branch_exception(#vst{current=#st{ct=[{_,[Fail]}|_]}}=Vst) + when is_integer(Fail) -> %% We have an active try/catch tag and we can jump there from this %% instruction, so we need to update the branched state of the try/catch %% handler. - valfun_3(I, branch_state(Fail, Vst)); -valfun_2(I, #vst{current=#st{ct=[]}}=Vst) -> - valfun_3(I, Vst); -valfun_2(_, _) -> + fork_state(Fail, Vst); +branch_exception(#vst{current=#st{ct=[]}}=Vst) -> + Vst; +branch_exception(_) -> error(ambiguous_catch_try_state). %% Handle the remaining floating point instructions here. %% Floating point. -valfun_3({fconv,Src,{fr,_}=Dst}, Vst) -> +valfun_2({fconv,Src,{fr,_}=Dst}, Vst) -> assert_term(Src, Vst), %% An exception is raised on error, hence branching to 0. @@ -635,261 +741,187 @@ valfun_3({fconv,Src,{fr,_}=Dst}, Vst) -> SuccVst = update_type(fun meet/2, number, Src, SuccVst0), set_freg(Dst, SuccVst) end); -valfun_3({bif,fadd,_,[_,_]=Ss,Dst}, Vst) -> +valfun_2({bif,fadd,_,[_,_]=Ss,Dst}, Vst) -> float_op(Ss, Dst, Vst); -valfun_3({bif,fdiv,_,[_,_]=Ss,Dst}, Vst) -> +valfun_2({bif,fdiv,_,[_,_]=Ss,Dst}, Vst) -> float_op(Ss, Dst, Vst); -valfun_3({bif,fmul,_,[_,_]=Ss,Dst}, Vst) -> +valfun_2({bif,fmul,_,[_,_]=Ss,Dst}, Vst) -> float_op(Ss, Dst, Vst); -valfun_3({bif,fnegate,_,[_]=Ss,Dst}, Vst) -> +valfun_2({bif,fnegate,_,[_]=Ss,Dst}, Vst) -> float_op(Ss, Dst, Vst); -valfun_3({bif,fsub,_,[_,_]=Ss,Dst}, Vst) -> +valfun_2({bif,fsub,_,[_,_]=Ss,Dst}, Vst) -> float_op(Ss, Dst, Vst); -valfun_3(fclearerror, Vst) -> +valfun_2(fclearerror, Vst) -> case get_fls(Vst) of - undefined -> ok; - checked -> ok; - Fls -> error({bad_floating_point_state,Fls}) + undefined -> ok; + checked -> ok; + Fls -> error({bad_floating_point_state,Fls}) end, set_fls(cleared, Vst); -valfun_3({fcheckerror,_}, Vst) -> +valfun_2({fcheckerror,_}, Vst) -> assert_fls(cleared, Vst), set_fls(checked, Vst); -valfun_3(I, Vst) -> - %% The instruction is not a float instruction. - case get_fls(Vst) of - undefined -> - valfun_4(I, Vst); - checked -> - valfun_4(I, Vst); - Fls -> - error({unsafe_instruction,{float_error_state,Fls}}) - end. +valfun_2(I, Vst) -> + assert_float_checked(Vst), + valfun_3(I, Vst). %% Instructions that can cause exceptions. -valfun_4({apply,Live}, Vst) -> - call(apply, Live+2, Vst); -valfun_4({apply_last,Live,_}, Vst) -> - tail_call(apply, Live+2, Vst); -valfun_4({call_fun,Live}, Vst) -> - validate_src([{x,Live}], Vst), - call('fun', Live+1, Vst); -valfun_4({call,Live,Func}, Vst) -> - call(Func, Live, Vst); -valfun_4({call_ext,Live,Func}, Vst) -> - %% Exception BIFs has already been taken care of above. - call(Func, Live, Vst); -valfun_4({call_only,Live,Func}, Vst) -> - tail_call(Func, Live, Vst); -valfun_4({call_ext_only,Live,Func}, Vst) -> - tail_call(Func, Live, Vst); -valfun_4({call_last,Live,Func,StkSize}, #vst{current=#st{numy=StkSize}}=Vst) -> - tail_call(Func, Live, Vst); -valfun_4({call_last,_,_,_}, #vst{current=#st{numy=NumY}}) -> - error({allocated,NumY}); -valfun_4({call_ext_last,Live,Func,StkSize}, - #vst{current=#st{numy=StkSize}}=Vst) -> - tail_call(Func, Live, Vst); -valfun_4({call_ext_last,_,_,_}, #vst{current=#st{numy=NumY}}) -> - error({allocated,NumY}); -valfun_4({make_fun2,_,_,_,Live}, Vst) -> - call(make_fun, Live, Vst); -%% Other BIFs -valfun_4({bif,element,{f,Fail},[Pos,Src],Dst}, Vst) -> - branch(Fail, Vst, - fun(SuccVst0) -> - PosType = get_term_type(Pos, SuccVst0), - TupleType = {tuple,[get_tuple_size(PosType)],#{}}, +valfun_3({make_fun2,{f,Lbl},_,_,NumFree}, #vst{ft=Ft}=Vst0) -> + #{ arity := Arity0 } = gb_trees:get(Lbl, Ft), + Arity = Arity0 - NumFree, - SuccVst1 = update_type(fun meet/2, TupleType, - Src, SuccVst0), - SuccVst = update_type(fun meet/2, {integer,[]}, - Pos, SuccVst1), + true = Arity >= 0, %Assertion. - ElementType = get_element_type(PosType, Src, SuccVst), - extract_term(ElementType, {bif,element}, [Pos,Src], - Dst, SuccVst) - end); -valfun_4({bif,raise,{f,0},Src,_Dst}, Vst) -> + Vst = prune_x_regs(NumFree, Vst0), + verify_call_args(make_fun, NumFree, Vst), + verify_y_init(Vst), + + create_term(#t_fun{arity=Arity}, make_fun, [], {x,0}, Vst); +%% Other BIFs +valfun_3({bif,raise,{f,0},Src,_Dst}, Vst) -> validate_src(Src, Vst), kill_state(Vst); -valfun_4(raw_raise=I, Vst) -> +valfun_3(raw_raise=I, Vst) -> call(I, 3, Vst); -valfun_4({bif,Op,{f,Fail},[Src]=Ss,Dst}, Vst) when Op =:= hd; Op =:= tl -> - assert_term(Src, Vst), - branch(Fail, Vst, - fun(FailVst) -> - update_type(fun subtract/2, cons, Src, FailVst) - end, - fun(SuccVst0) -> - SuccVst = update_type(fun meet/2, cons, Src, SuccVst0), - extract_term(term, {bif,Op}, Ss, Dst, SuccVst) - end); -valfun_4({bif,Op,{f,Fail},Ss,Dst}, Vst) -> +valfun_3({bif,Op,{f,Fail},Ss,Dst}, Vst) -> validate_src(Ss, Vst), - branch(Fail, Vst, - fun(SuccVst0) -> - %% Infer argument types. Note that we can't subtract - %% types as the BIF could fail for reasons other than - %% bad argument types. - ArgTypes = bif_arg_types(Op, Ss), - SuccVst = foldl(fun({Arg, T}, V) -> - update_type(fun meet/2, T, Arg, V) - end, SuccVst0, zip(Ss, ArgTypes)), - Type = bif_return_type(Op, Ss, SuccVst), - extract_term(Type, {bif,Op}, Ss, Dst, SuccVst) - end); -valfun_4({gc_bif,Op,{f,Fail},Live,Ss,Dst}, #vst{current=St0}=Vst0) -> - validate_src(Ss, Vst0), - verify_live(Live, Vst0), - verify_y_init(Vst0), - - %% Heap allocations and X registers are killed regardless of whether we - %% fail or not, as we may fail after GC. - St = kill_heap_allocation(St0), - Vst = prune_x_regs(Live, Vst0#vst{current=St}), - - branch(Fail, Vst, - fun(SuccVst0) -> - ArgTypes = bif_arg_types(Op, Ss), - SuccVst = foldl(fun({Arg, T}, V) -> - update_type(fun meet/2, T, Arg, V) - end, SuccVst0, zip(Ss, ArgTypes)), - - Type = bif_return_type(Op, Ss, SuccVst), - - %% We're passing Vst0 as the original because the - %% registers were pruned before the branch. - extract_term(Type, {gc_bif,Op}, Ss, Dst, SuccVst, Vst0) - end); -valfun_4(return, #vst{current=#st{numy=none}}=Vst) -> + validate_bif(Op, Fail, Ss, Dst, Vst); +valfun_3({gc_bif,Op,{f,Fail},Live,Ss,Dst}, Vst) -> + validate_gc_bif(Op, Fail, Ss, Dst, Live, Vst); +valfun_3(return, #vst{current=#st{numy=none}}=Vst) -> assert_durable_term({x,0}, Vst), kill_state(Vst); -valfun_4(return, #vst{current=#st{numy=NumY}}) -> +valfun_3(return, #vst{current=#st{numy=NumY}}) -> error({stack_frame,NumY}); -valfun_4({loop_rec,{f,Fail},Dst}, Vst) -> +valfun_3({loop_rec,{f,Fail},Dst}, Vst) -> %% This term may not be part of the root set until remove_message/0 is %% executed. If control transfers to the loop_rec_end/1 instruction, no %% part of this term must be stored in a Y register. branch(Fail, Vst, fun(SuccVst0) -> - {Ref, SuccVst} = new_value(term, loop_rec, [], SuccVst0), + {Ref, SuccVst} = new_value(any, loop_rec, [], SuccVst0), mark_fragile(Dst, set_reg_vref(Ref, Dst, SuccVst)) end); -valfun_4({wait,_}, Vst) -> +valfun_3({wait,_}, Vst) -> verify_y_init(Vst), kill_state(Vst); -valfun_4({wait_timeout,_,Src}, Vst) -> +valfun_3({wait_timeout,_,Src}, Vst) -> assert_term(Src, Vst), verify_y_init(Vst), prune_x_regs(0, Vst); -valfun_4({loop_rec_end,_}, Vst) -> +valfun_3({loop_rec_end,_}, Vst) -> verify_y_init(Vst), kill_state(Vst); -valfun_4(timeout, Vst) -> +valfun_3(timeout, Vst) -> prune_x_regs(0, Vst); -valfun_4(send, Vst) -> +valfun_3(send, Vst) -> call(send, 2, Vst); -valfun_4({set_tuple_element,Src,Tuple,N}, Vst) -> - I = N + 1, - assert_term(Src, Vst), - assert_type({tuple_element,I}, Tuple, Vst), - %% Manually update the tuple type; we can't rely on the ordinary update - %% helpers as we must support overwriting (rather than just widening or - %% narrowing) known elements, and we can't use extract_term either since - %% the source tuple may be aliased. - {tuple, Sz, Es0} = get_term_type(Tuple, Vst), - Es = set_element_type({integer,I}, get_term_type(Src, Vst), Es0), - override_type({tuple, Sz, Es}, Tuple, Vst); %% Match instructions. -valfun_4({select_val,Src,{f,Fail},{list,Choices}}, Vst) -> +valfun_3({select_val,Src,{f,Fail},{list,Choices}}, Vst) -> assert_term(Src, Vst), assert_choices(Choices), validate_select_val(Fail, Choices, Src, Vst); -valfun_4({select_tuple_arity,Tuple,{f,Fail},{list,Choices}}, Vst) -> - assert_type(tuple, Tuple, Vst), +valfun_3({select_tuple_arity,Tuple,{f,Fail},{list,Choices}}, Vst) -> + assert_type(#t_tuple{}, Tuple, Vst), assert_arities(Choices), validate_select_tuple_arity(Fail, Choices, Tuple, Vst); %% New bit syntax matching instructions. -valfun_4({test,bs_start_match3,{f,Fail},Live,[Src],Dst}, Vst) -> +valfun_3({test,bs_start_match3,{f,Fail},Live,[Src],Dst}, Vst) -> validate_bs_start_match(Fail, Live, bsm_match_state(), Src, Dst, Vst); -valfun_4({test,bs_start_match2,{f,Fail},Live,[Src,Slots],Dst}, Vst) -> +valfun_3({test,bs_start_match2,{f,Fail},Live,[Src,Slots],Dst}, Vst) -> validate_bs_start_match(Fail, Live, bsm_match_state(Slots), Src, Dst, Vst); -valfun_4({test,bs_match_string,{f,Fail},[Ctx,_,_]}, Vst) -> - bsm_validate_context(Ctx, Vst), +valfun_3({test,bs_match_string,{f,Fail},[Ctx,_,_]}, Vst) -> + assert_type(#t_bs_context{}, Ctx, Vst), branch(Fail, Vst, fun(V) -> V end); -valfun_4({test,bs_skip_bits2,{f,Fail},[Ctx,Src,_,_]}, Vst) -> - bsm_validate_context(Ctx, Vst), +valfun_3({test,bs_skip_bits2,{f,Fail},[Ctx,Src,_,_]}, Vst) -> + assert_type(#t_bs_context{}, Ctx, Vst), assert_term(Src, Vst), branch(Fail, Vst, fun(V) -> V end); -valfun_4({test,bs_test_tail2,{f,Fail},[Ctx,_]}, Vst) -> - bsm_validate_context(Ctx, Vst), +valfun_3({test,bs_test_tail2,{f,Fail},[Ctx,_]}, Vst) -> + assert_type(#t_bs_context{}, Ctx, Vst), branch(Fail, Vst, fun(V) -> V end); -valfun_4({test,bs_test_unit,{f,Fail},[Ctx,_]}, Vst) -> - bsm_validate_context(Ctx, Vst), +valfun_3({test,bs_test_unit,{f,Fail},[Ctx,_]}, Vst) -> + assert_type(#t_bs_context{}, Ctx, Vst), branch(Fail, Vst, fun(V) -> V end); -valfun_4({test,bs_skip_utf8,{f,Fail},[Ctx,Live,_]}, Vst) -> +valfun_3({test,bs_skip_utf8,{f,Fail},[Ctx,Live,_]}, Vst) -> validate_bs_skip_utf(Fail, Ctx, Live, Vst); -valfun_4({test,bs_skip_utf16,{f,Fail},[Ctx,Live,_]}, Vst) -> +valfun_3({test,bs_skip_utf16,{f,Fail},[Ctx,Live,_]}, Vst) -> validate_bs_skip_utf(Fail, Ctx, Live, Vst); -valfun_4({test,bs_skip_utf32,{f,Fail},[Ctx,Live,_]}, Vst) -> +valfun_3({test,bs_skip_utf32,{f,Fail},[Ctx,Live,_]}, Vst) -> validate_bs_skip_utf(Fail, Ctx, Live, Vst); -valfun_4({test,bs_get_integer2=Op,{f,Fail},Live,[Ctx,_,_,_],Dst}, Vst) -> - validate_bs_get(Op, Fail, Ctx, Live, {integer, []}, Dst, Vst); -valfun_4({test,bs_get_float2=Op,{f,Fail},Live,[Ctx,_,_,_],Dst}, Vst) -> - validate_bs_get(Op, Fail, Ctx, Live, {float, []}, Dst, Vst); -valfun_4({test,bs_get_binary2=Op,{f,Fail},Live,[Ctx,_,_,_],Dst}, Vst) -> - validate_bs_get(Op, Fail, Ctx, Live, binary, Dst, Vst); -valfun_4({test,bs_get_utf8=Op,{f,Fail},Live,[Ctx,_],Dst}, Vst) -> - validate_bs_get(Op, Fail, Ctx, Live, {integer, []}, Dst, Vst); -valfun_4({test,bs_get_utf16=Op,{f,Fail},Live,[Ctx,_],Dst}, Vst) -> - validate_bs_get(Op, Fail, Ctx, Live, {integer, []}, Dst, Vst); -valfun_4({test,bs_get_utf32=Op,{f,Fail},Live,[Ctx,_],Dst}, Vst) -> - validate_bs_get(Op, Fail, Ctx, Live, {integer, []}, Dst, Vst); -valfun_4({bs_save2,Ctx,SavePoint}, Vst) -> +valfun_3({test,bs_get_integer2=Op,{f,Fail},Live, + [Ctx,{integer,Size},Unit,{field_flags,Flags}],Dst},Vst) + when Size * Unit =< 64 -> + Type = case member(unsigned, Flags) of + true -> + NumBits = Size * Unit, + beam_types:make_integer(0, (1 bsl NumBits)-1); + false -> + %% Signed integer or way too large, don't bother. + #t_integer{} + end, + validate_bs_get(Op, Fail, Ctx, Live, Type, Dst, Vst); +valfun_3({test,bs_get_integer2=Op,{f,Fail},Live, + [Ctx,_Size,_Unit,_Flags],Dst},Vst) -> + validate_bs_get(Op, Fail, Ctx, Live, #t_integer{}, Dst, Vst); +valfun_3({test,bs_get_float2=Op,{f,Fail},Live,[Ctx,_,_,_],Dst}, Vst) -> + validate_bs_get(Op, Fail, Ctx, Live, float, Dst, Vst); +valfun_3({test,bs_get_binary2=Op,{f,Fail},Live,[Ctx,_,Unit,_],Dst}, Vst) -> + validate_bs_get(Op, Fail, Ctx, Live, #t_bitstring{unit=Unit}, Dst, Vst); +valfun_3({test,bs_get_utf8=Op,{f,Fail},Live,[Ctx,_],Dst}, Vst) -> + Type = beam_types:make_integer(0, ?UNICODE_MAX), + validate_bs_get(Op, Fail, Ctx, Live, Type, Dst, Vst); +valfun_3({test,bs_get_utf16=Op,{f,Fail},Live,[Ctx,_],Dst}, Vst) -> + Type = beam_types:make_integer(0, ?UNICODE_MAX), + validate_bs_get(Op, Fail, Ctx, Live, Type, Dst, Vst); +valfun_3({test,bs_get_utf32=Op,{f,Fail},Live,[Ctx,_],Dst}, Vst) -> + Type = beam_types:make_integer(0, ?UNICODE_MAX), + validate_bs_get(Op, Fail, Ctx, Live, Type, Dst, Vst); +valfun_3({bs_save2,Ctx,SavePoint}, Vst) -> bsm_save(Ctx, SavePoint, Vst); -valfun_4({bs_restore2,Ctx,SavePoint}, Vst) -> +valfun_3({bs_restore2,Ctx,SavePoint}, Vst) -> bsm_restore(Ctx, SavePoint, Vst); -valfun_4({bs_get_position, Ctx, Dst, Live}, Vst0) -> - bsm_validate_context(Ctx, Vst0), +valfun_3({bs_get_position, Ctx, Dst, Live}, Vst0) -> + assert_type(#t_bs_context{}, Ctx, Vst0), verify_live(Live, Vst0), verify_y_init(Vst0), Vst = prune_x_regs(Live, Vst0), - create_term(ms_position, bs_get_position, [Ctx], Dst, Vst, Vst0); -valfun_4({bs_set_position, Ctx, Pos}, Vst) -> - bsm_validate_context(Ctx, Vst), - assert_type(ms_position, Pos, Vst), + create_term(#t_abstract{kind=ms_position}, bs_get_position, [Ctx], + Dst, Vst, Vst0); +valfun_3({bs_set_position, Ctx, Pos}, Vst) -> + assert_type(#t_bs_context{}, Ctx, Vst), + assert_type(#t_abstract{kind=ms_position}, Pos, Vst), Vst; %% Other test instructions. -valfun_4({test,has_map_fields,{f,Lbl},Src,{list,List}}, Vst) -> - assert_type(map, Src, Vst), +valfun_3({test,has_map_fields,{f,Lbl},Src,{list,List}}, Vst) -> + assert_type(#t_map{}, Src, Vst), assert_unique_map_keys(List), branch(Lbl, Vst, fun(V) -> V end); -valfun_4({test,is_atom,{f,Lbl},[Src]}, Vst) -> - type_test(Lbl, {atom,[]}, Src, Vst); -valfun_4({test,is_binary,{f,Lbl},[Src]}, Vst) -> - type_test(Lbl, binary, Src, Vst); -valfun_4({test,is_bitstr,{f,Lbl},[Src]}, Vst) -> - type_test(Lbl, binary, Src, Vst); -valfun_4({test,is_boolean,{f,Lbl},[Src]}, Vst) -> - type_test(Lbl, bool, Src, Vst); -valfun_4({test,is_float,{f,Lbl},[Src]}, Vst) -> - type_test(Lbl, {float,[]}, Src, Vst); -valfun_4({test,is_tuple,{f,Lbl},[Src]}, Vst) -> - type_test(Lbl, {tuple,[0],#{}}, Src, Vst); -valfun_4({test,is_integer,{f,Lbl},[Src]}, Vst) -> - type_test(Lbl, {integer,[]}, Src, Vst); -valfun_4({test,is_nonempty_list,{f,Lbl},[Src]}, Vst) -> +valfun_3({test,is_atom,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, #t_atom{}, Src, Vst); +valfun_3({test,is_binary,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, #t_bitstring{unit=8}, Src, Vst); +valfun_3({test,is_bitstr,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, #t_bitstring{}, Src, Vst); +valfun_3({test,is_boolean,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, beam_types:make_boolean(), Src, Vst); +valfun_3({test,is_float,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, float, Src, Vst); +valfun_3({test,is_tuple,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, #t_tuple{}, Src, Vst); +valfun_3({test,is_integer,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, #t_integer{}, Src, Vst); +valfun_3({test,is_nonempty_list,{f,Lbl},[Src]}, Vst) -> type_test(Lbl, cons, Src, Vst); -valfun_4({test,is_number,{f,Lbl},[Src]}, Vst) -> +valfun_3({test,is_number,{f,Lbl},[Src]}, Vst) -> type_test(Lbl, number, Src, Vst); -valfun_4({test,is_list,{f,Lbl},[Src]}, Vst) -> +valfun_3({test,is_list,{f,Lbl},[Src]}, Vst) -> type_test(Lbl, list, Src, Vst); -valfun_4({test,is_map,{f,Lbl},[Src]}, Vst) -> - type_test(Lbl, map, Src, Vst); -valfun_4({test,is_nil,{f,Lbl},[Src]}, Vst) -> +valfun_3({test,is_map,{f,Lbl},[Src]}, Vst) -> + type_test(Lbl, #t_map{}, Src, Vst); +valfun_3({test,is_nil,{f,Lbl},[Src]}, Vst) -> %% is_nil is an exact check against the 'nil' value, and should not be %% treated as a simple type test. assert_term(Src, Vst), @@ -900,15 +932,16 @@ valfun_4({test,is_nil,{f,Lbl},[Src]}, Vst) -> fun(SuccVst) -> update_eq_types(Src, nil, SuccVst) end); -valfun_4({test,test_arity,{f,Lbl},[Tuple,Sz]}, Vst) when is_integer(Sz) -> - assert_type(tuple, Tuple, Vst), - Type = {tuple, Sz, #{}}, +valfun_3({test,test_arity,{f,Lbl},[Tuple,Sz]}, Vst) when is_integer(Sz) -> + assert_type(#t_tuple{}, Tuple, Vst), + Type = #t_tuple{exact=true,size=Sz}, type_test(Lbl, Type, Tuple, Vst); -valfun_4({test,is_tagged_tuple,{f,Lbl},[Src,Sz,Atom]}, Vst) -> +valfun_3({test,is_tagged_tuple,{f,Lbl},[Src,Sz,Atom]}, Vst) -> assert_term(Src, Vst), - Type = {tuple, Sz, #{ {integer,1} => Atom }}, + Es = #{ 1 => get_literal_type(Atom) }, + Type = #t_tuple{exact=true,size=Sz,elements=Es}, type_test(Lbl, Type, Src, Vst); -valfun_4({test,is_eq_exact,{f,Lbl},[Src,Val]=Ss}, Vst) -> +valfun_3({test,is_eq_exact,{f,Lbl},[Src,Val]=Ss}, Vst) -> validate_src(Ss, Vst), branch(Lbl, Vst, fun(FailVst) -> @@ -917,7 +950,7 @@ valfun_4({test,is_eq_exact,{f,Lbl},[Src,Val]=Ss}, Vst) -> fun(SuccVst) -> update_eq_types(Src, Val, SuccVst) end); -valfun_4({test,is_ne_exact,{f,Lbl},[Src,Val]=Ss}, Vst) -> +valfun_3({test,is_ne_exact,{f,Lbl},[Src,Val]=Ss}, Vst) -> validate_src(Ss, Vst), branch(Lbl, Vst, fun(FailVst) -> @@ -926,30 +959,30 @@ valfun_4({test,is_ne_exact,{f,Lbl},[Src,Val]=Ss}, Vst) -> fun(SuccVst) -> update_ne_types(Src, Val, SuccVst) end); -valfun_4({test,_Op,{f,Lbl},Src}, Vst) -> +valfun_3({test,_Op,{f,Lbl},Src}, Vst) -> %% is_pid, is_reference, et cetera. validate_src(Src, Vst), branch(Lbl, Vst, fun(V) -> V end); -valfun_4({bs_add,{f,Fail},[A,B,_],Dst}, Vst) -> +valfun_3({bs_add,{f,Fail},[A,B,_],Dst}, Vst) -> assert_term(A, Vst), assert_term(B, Vst), branch(Fail, Vst, fun(SuccVst) -> - create_term({integer,[]}, bs_add, [A, B], Dst, SuccVst) + create_term(#t_integer{}, bs_add, [A, B], Dst, SuccVst) end); -valfun_4({bs_utf8_size,{f,Fail},A,Dst}, Vst) -> +valfun_3({bs_utf8_size,{f,Fail},A,Dst}, Vst) -> assert_term(A, Vst), branch(Fail, Vst, fun(SuccVst) -> - create_term({integer,[]}, bs_utf8_size, [A], Dst, SuccVst) + create_term(#t_integer{}, bs_utf8_size, [A], Dst, SuccVst) end); -valfun_4({bs_utf16_size,{f,Fail},A,Dst}, Vst) -> +valfun_3({bs_utf16_size,{f,Fail},A,Dst}, Vst) -> assert_term(A, Vst), branch(Fail, Vst, fun(SuccVst) -> - create_term({integer,[]}, bs_utf16_size, [A], Dst, SuccVst) + create_term(#t_integer{}, bs_utf16_size, [A], Dst, SuccVst) end); -valfun_4({bs_init2,{f,Fail},Sz,Heap,Live,_,Dst}, Vst0) -> +valfun_3({bs_init2,{f,Fail},Sz,Heap,Live,_,Dst}, Vst0) -> verify_live(Live, Vst0), verify_y_init(Vst0), if @@ -962,9 +995,10 @@ valfun_4({bs_init2,{f,Fail},Sz,Heap,Live,_,Dst}, Vst0) -> branch(Fail, Vst, fun(SuccVst0) -> SuccVst = prune_x_regs(Live, SuccVst0), - create_term(binary, bs_init2, [], Dst, SuccVst, SuccVst0) + create_term(#t_bitstring{unit=8}, bs_init2, [], Dst, + SuccVst, SuccVst0) end); -valfun_4({bs_init_bits,{f,Fail},Sz,Heap,Live,_,Dst}, Vst0) -> +valfun_3({bs_init_bits,{f,Fail},Sz,Heap,Live,_,Dst}, Vst0) -> verify_live(Live, Vst0), verify_y_init(Vst0), if @@ -977,9 +1011,9 @@ valfun_4({bs_init_bits,{f,Fail},Sz,Heap,Live,_,Dst}, Vst0) -> branch(Fail, Vst, fun(SuccVst0) -> SuccVst = prune_x_regs(Live, SuccVst0), - create_term(binary, bs_init_bits, [], Dst, SuccVst) + create_term(#t_bitstring{}, bs_init_bits, [], Dst, SuccVst) end); -valfun_4({bs_append,{f,Fail},Bits,Heap,Live,_Unit,Bin,_Flags,Dst}, Vst0) -> +valfun_3({bs_append,{f,Fail},Bits,Heap,Live,Unit,Bin,_Flags,Dst}, Vst0) -> verify_live(Live, Vst0), verify_y_init(Vst0), assert_term(Bits, Vst0), @@ -988,69 +1022,71 @@ valfun_4({bs_append,{f,Fail},Bits,Heap,Live,_Unit,Bin,_Flags,Dst}, Vst0) -> branch(Fail, Vst, fun(SuccVst0) -> SuccVst = prune_x_regs(Live, SuccVst0), - create_term(binary, bs_append, [Bin], Dst, SuccVst, SuccVst0) + create_term(#t_bitstring{unit=Unit}, bs_append, + [Bin], Dst, SuccVst, SuccVst0) end); -valfun_4({bs_private_append,{f,Fail},Bits,_Unit,Bin,_Flags,Dst}, Vst) -> +valfun_3({bs_private_append,{f,Fail},Bits,Unit,Bin,_Flags,Dst}, Vst) -> assert_term(Bits, Vst), assert_term(Bin, Vst), branch(Fail, Vst, fun(SuccVst) -> - create_term(binary, bs_private_append, [Bin], Dst, SuccVst) + create_term(#t_bitstring{unit=Unit}, bs_private_append, + [Bin], Dst, SuccVst) end); -valfun_4({bs_put_string,Sz,_}, Vst) when is_integer(Sz) -> +valfun_3({bs_put_string,Sz,_}, Vst) when is_integer(Sz) -> Vst; -valfun_4({bs_put_binary,{f,Fail},Sz,_,_,Src}, Vst) -> +valfun_3({bs_put_binary,{f,Fail},Sz,_,_,Src}, Vst) -> assert_term(Sz, Vst), assert_term(Src, Vst), branch(Fail, Vst, fun(SuccVst) -> - update_type(fun meet/2, binary, Src, SuccVst) + update_type(fun meet/2, #t_bitstring{}, Src, SuccVst) end); -valfun_4({bs_put_float,{f,Fail},Sz,_,_,Src}, Vst) -> +valfun_3({bs_put_float,{f,Fail},Sz,_,_,Src}, Vst) -> assert_term(Sz, Vst), assert_term(Src, Vst), branch(Fail, Vst, fun(SuccVst) -> - update_type(fun meet/2, {float,[]}, Src, SuccVst) + update_type(fun meet/2, float, Src, SuccVst) end); -valfun_4({bs_put_integer,{f,Fail},Sz,_,_,Src}, Vst) -> +valfun_3({bs_put_integer,{f,Fail},Sz,_,_,Src}, Vst) -> assert_term(Sz, Vst), assert_term(Src, Vst), branch(Fail, Vst, fun(SuccVst) -> - update_type(fun meet/2, {integer,[]}, Src, SuccVst) + update_type(fun meet/2, #t_integer{}, Src, SuccVst) end); -valfun_4({bs_put_utf8,{f,Fail},_,Src}, Vst) -> +valfun_3({bs_put_utf8,{f,Fail},_,Src}, Vst) -> assert_term(Src, Vst), branch(Fail, Vst, fun(SuccVst) -> - update_type(fun meet/2, {integer,[]}, Src, SuccVst) + update_type(fun meet/2, #t_integer{}, Src, SuccVst) end); -valfun_4({bs_put_utf16,{f,Fail},_,Src}, Vst) -> +valfun_3({bs_put_utf16,{f,Fail},_,Src}, Vst) -> assert_term(Src, Vst), branch(Fail, Vst, fun(SuccVst) -> - update_type(fun meet/2, {integer,[]}, Src, SuccVst) + update_type(fun meet/2, #t_integer{}, Src, SuccVst) end); -valfun_4({bs_put_utf32,{f,Fail},_,Src}, Vst) -> +valfun_3({bs_put_utf32,{f,Fail},_,Src}, Vst) -> assert_term(Src, Vst), branch(Fail, Vst, fun(SuccVst) -> - update_type(fun meet/2, {integer,[]}, Src, SuccVst) + update_type(fun meet/2, #t_integer{}, Src, SuccVst) end); %% Map instructions. -valfun_4({put_map_assoc=Op,{f,Fail},Src,Dst,Live,{list,List}}, Vst) -> +valfun_3({put_map_assoc=Op,{f,Fail},Src,Dst,Live,{list,List}}, Vst) -> verify_put_map(Op, Fail, Src, Dst, Live, List, Vst); -valfun_4({put_map_exact=Op,{f,Fail},Src,Dst,Live,{list,List}}, Vst) -> +valfun_3({put_map_exact=Op,{f,Fail},Src,Dst,Live,{list,List}}, Vst) -> verify_put_map(Op, Fail, Src, Dst, Live, List, Vst); -valfun_4({get_map_elements,{f,Fail},Src,{list,List}}, Vst) -> +valfun_3({get_map_elements,{f,Fail},Src,{list,List}}, Vst) -> verify_get_map(Fail, Src, List, Vst); -valfun_4(_, _) -> +valfun_3(_, _) -> error(unknown_instruction). verify_get_map(Fail, Src, List, Vst0) -> assert_not_literal(Src), %OTP 22. - assert_type(map, Src, Vst0), + assert_type(#t_map{}, Src, Vst0), branch(Fail, Vst0, fun(FailVst) -> @@ -1074,7 +1110,7 @@ verify_get_map(Fail, Src, List, Vst0) -> clobber_map_vals([Key,Dst|T], Map, Vst0) -> case is_reg_initialized(Dst, Vst0) of true -> - Vst = extract_term(term, {bif,map_get}, [Key, Map], Dst, Vst0), + Vst = extract_term(any, {bif,map_get}, [Key, Map], Dst, Vst0), clobber_map_vals(T, Map, Vst); false -> clobber_map_vals(T, Map, Vst0) @@ -1099,13 +1135,13 @@ extract_map_keys([]) -> []. extract_map_vals([Key,Dst|Vs], Map, Vst0, Vsti0) -> assert_term(Key, Vst0), - Vsti = extract_term(term, {bif,map_get}, [Key, Map], Dst, Vsti0), + Vsti = extract_term(any, {bif,map_get}, [Key, Map], Dst, Vsti0), extract_map_vals(Vs, Map, Vst0, Vsti); extract_map_vals([], _Map, _Vst0, Vst) -> Vst. verify_put_map(Op, Fail, Src, Dst, Live, List, Vst0) -> - assert_type(map, Src, Vst0), + assert_type(#t_map{}, Src, Vst0), verify_live(Live, Vst0), verify_y_init(Vst0), _ = [assert_term(Term, Vst0) || Term <- List], @@ -1116,10 +1152,73 @@ verify_put_map(Op, Fail, Src, Dst, Live, List, Vst0) -> SuccVst = prune_x_regs(Live, SuccVst0), Keys = extract_map_keys(List), assert_unique_map_keys(Keys), - create_term(map, Op, [Src], Dst, SuccVst, SuccVst0) + create_term(#t_map{}, Op, [Src], Dst, SuccVst, SuccVst0) end). %% +%% Common code for validating BIFs. +%% +%% OrigVst is the state we entered the instruction with, which is needed for +%% gc_bifs as X registers are pruned prior to calling this function, which may +%% have clobbered the sources. +%% + +validate_bif(Op, Fail, Ss, Dst, Vst) -> + validate_src(Ss, Vst), + validate_bif_1(bif, Op, Fail, Ss, Dst, Vst, Vst). + +validate_gc_bif(Op, Fail, Ss, Dst, Live, #vst{current=St0}=Vst0) -> + validate_src(Ss, Vst0), + verify_live(Live, Vst0), + verify_y_init(Vst0), + + %% Heap allocations and X registers are killed regardless of whether we + %% fail or not, as we may fail after GC. + St = kill_heap_allocation(St0), + Vst = prune_x_regs(Live, Vst0#vst{current=St}), + validate_src(Ss, Vst), + + validate_bif_1(gc_bif, Op, Fail, Ss, Dst, Vst, Vst). + +validate_bif_1(Kind, Op, cannot_fail, Ss, Dst, OrigVst, Vst0) -> + %% This BIF explicitly cannot fail; it will not jump to a guard nor throw + %% an exception. Validation will fail if it returns 'none' or has a type + %% conflict on one of its arguments. + + {Type, ArgTypes, _CanSubtract} = bif_types(Op, Ss, Vst0), + ZippedArgs = zip(Ss, ArgTypes), + + Vst = foldl(fun({A, T}, V) -> + update_type(fun meet/2, T, A, V) + end, Vst0, ZippedArgs), + + true = Type =/= none, %Assertion. + + extract_term(Type, {Kind, Op}, Ss, Dst, Vst, OrigVst); +validate_bif_1(Kind, Op, Fail, Ss, Dst, OrigVst, Vst) -> + {Type, ArgTypes, CanSubtract} = bif_types(Op, Ss, Vst), + ZippedArgs = zip(Ss, ArgTypes), + + FailFun = case CanSubtract of + true -> + fun(FailVst0) -> + foldl(fun({A, T}, V) -> + update_type(fun subtract/2, T, A, V) + end, FailVst0, ZippedArgs) + end; + false -> + fun(S) -> S end + end, + SuccFun = fun(SuccVst0) -> + SuccVst = foldl(fun({A, T}, V) -> + update_type(fun meet/2, T, A, V) + end, SuccVst0, ZippedArgs), + extract_term(Type, {Kind, Op}, Ss, Dst, SuccVst, OrigVst) + end, + + branch(Fail, Vst, FailFun, SuccFun). + +%% %% Common code for validating bs_start_match* instructions. %% @@ -1127,18 +1226,18 @@ validate_bs_start_match(Fail, Live, Type, Src, Dst, Vst) -> verify_live(Live, Vst), verify_y_init(Vst), - %% #ms{} can represent either a match context or a term, so we have to mark - %% the source as a term if it fails with a match context as an input. This - %% hack is only needed until we get proper union types. + %% #t_bs_context{} can represent either a match context or a term, so we + %% have to mark the source as a term if it fails with a match context as an + %% input. This hack is only needed until we get proper union types. branch(Fail, Vst, fun(FailVst) -> case get_movable_term_type(Src, FailVst) of - #ms{} -> override_type(term, Src, FailVst); + #t_bs_context{} -> override_type(any, Src, FailVst); _ -> FailVst end end, fun(SuccVst0) -> - SuccVst1 = update_type(fun meet/2, binary, + SuccVst1 = update_type(fun meet/2, #t_bitstring{}, Src, SuccVst0), SuccVst = prune_x_regs(Live, SuccVst1), extract_term(Type, bs_start_match, [Src], Dst, @@ -1149,7 +1248,7 @@ validate_bs_start_match(Fail, Live, Type, Src, Dst, Vst) -> %% Common code for validating bs_get* instructions. %% validate_bs_get(Op, Fail, Ctx, Live, Type, Dst, Vst) -> - bsm_validate_context(Ctx, Vst), + assert_type(#t_bs_context{}, Ctx, Vst), verify_live(Live, Vst), verify_y_init(Vst), @@ -1163,7 +1262,7 @@ validate_bs_get(Op, Fail, Ctx, Live, Type, Dst, Vst) -> %% Common code for validating bs_skip_utf* instructions. %% validate_bs_skip_utf(Fail, Ctx, Live, Vst) -> - bsm_validate_context(Ctx, Vst), + assert_type(#t_bs_context{}, Ctx, Vst), verify_y_init(Vst), verify_live(Live, Vst), @@ -1217,12 +1316,13 @@ kill_state(Vst) -> call(Name, Live, #vst{current=St0}=Vst0) -> verify_call_args(Name, Live, Vst0), verify_y_init(Vst0), - case call_return_type(Name, Vst0) of - Type when Type =/= exception -> - %% Type is never 'exception' because it has been handled earlier. + case call_types(Name, Live, Vst0) of + {none, _, _} -> + kill_state(Vst0); + {RetType, _, _} -> St = St0#st{f=init_fregs()}, Vst = prune_x_regs(0, Vst0#vst{current=St}), - create_term(Type, call, [], {x,0}, Vst) + create_term(RetType, call, [], {x,0}, Vst) end. %% Tail call. @@ -1237,8 +1337,15 @@ tail_call(Name, Live, Vst0) -> verify_call_args(_, 0, #vst{}) -> ok; -verify_call_args({f,Lbl}, Live, Vst) when is_integer(Live)-> - verify_local_args(Live - 1, Lbl, #{}, Vst); +verify_call_args({f,Lbl}, Live, #vst{ft=Ft}=Vst) when is_integer(Live) -> + case gb_trees:lookup(Lbl, Ft) of + {value, FuncInfo} -> + #{ arity := Live, + parameter_types := ParamTypes } = FuncInfo, + verify_local_args(Live - 1, ParamTypes, #{}, Vst); + none -> + error(local_call_to_unknown_function) + end; verify_call_args(_, Live, Vst) when is_integer(Live)-> verify_remote_args_1(Live - 1, Vst); verify_call_args(_, Live, _) -> @@ -1250,87 +1357,50 @@ verify_remote_args_1(X, Vst) -> assert_durable_term({x, X}, Vst), verify_remote_args_1(X - 1, Vst). -verify_local_args(-1, _Lbl, _CtxIds, _Vst) -> +verify_local_args(-1, _ParamTypes, _CtxIds, _Vst) -> ok; -verify_local_args(X, Lbl, CtxIds, Vst) -> +verify_local_args(X, ParamTypes, CtxRefs, Vst) -> Reg = {x, X}, assert_not_fragile(Reg, Vst), case get_movable_term_type(Reg, Vst) of - #ms{id=Id}=Type -> - case CtxIds of - #{ Id := Other } -> + #t_bs_context{}=Type -> + VRef = get_reg_vref(Reg, Vst), + case CtxRefs of + #{ VRef := Other } -> error({multiple_match_contexts, [Reg, Other]}); #{} -> - verify_arg_type(Lbl, Reg, Type, Vst), - verify_local_args(X - 1, Lbl, CtxIds#{ Id => Reg }, Vst) + verify_arg_type(Reg, Type, ParamTypes), + verify_local_args(X - 1, ParamTypes, + CtxRefs#{ VRef => Reg }, Vst) end; Type -> - verify_arg_type(Lbl, Reg, Type, Vst), - verify_local_args(X - 1, Lbl, CtxIds, Vst) + verify_arg_type(Reg, Type, ParamTypes), + verify_local_args(X - 1, ParamTypes, CtxRefs, Vst) end. %% Verifies that the given argument narrows to what the function expects. -verify_arg_type(Lbl, Reg, #ms{}, #vst{ft=Ft}) -> +verify_arg_type(Reg, #t_bs_context{}, ParamTypes) -> %% Match contexts require explicit support, and may not be passed to a %% function that accepts arbitrary terms. - case gb_trees:lookup({Lbl, Reg}, Ft) of - {value, #ms{}} -> ok; - _ -> error(no_bs_start_match2) + case ParamTypes of + #{ Reg := #t_bs_context{}} -> ok; + #{} -> error(no_bs_start_match2) end; -verify_arg_type(Lbl, Reg, GivenType, #vst{ft=Ft}) -> - case gb_trees:lookup({Lbl, Reg}, Ft) of - {value, #ms{}} -> +verify_arg_type(Reg, GivenType, ParamTypes) -> + case ParamTypes of + #{ Reg := #t_bs_context{}} -> %% Functions that accept match contexts also accept all other %% terms. This will change once we support union types. ok; - {value, RequiredType} -> - case vat_1(GivenType, RequiredType) of - true -> ok; - false -> error({bad_arg_type, Reg, GivenType, RequiredType}) + #{ Reg := RequiredType } -> + case meet(GivenType, RequiredType) of + GivenType -> ok; + _ -> error({bad_arg_type, Reg, GivenType, RequiredType}) end; - none -> + #{} -> ok end. -%% Checks whether the Given argument is compatible with the Required one. This -%% is essentially a relaxed version of 'meet(Given, Req) =:= Given', where we -%% accept that the Given value has the right type but not necessarily the exact -%% same value; if {atom,gurka} is required, we'll consider {atom,[]} valid. -%% -%% This will catch all problems that could crash the emulator, like passing a -%% 1-tuple when the callee expects a 3-tuple, but some value errors might slip -%% through. -vat_1(Same, Same) -> true; -vat_1({atom,A}, {atom,B}) -> A =:= B orelse is_list(A) orelse is_list(B); -vat_1({atom,A}, bool) -> is_boolean(A) orelse is_list(A); -vat_1(bool, {atom,B}) -> is_boolean(B) orelse is_list(B); -vat_1(cons, list) -> true; -vat_1({float,A}, {float,B}) -> A =:= B orelse is_list(A) orelse is_list(B); -vat_1({float,_}, number) -> true; -vat_1({integer,A}, {integer,B}) -> A =:= B orelse is_list(A) orelse is_list(B); -vat_1({integer,_}, number) -> true; -vat_1(_, {literal,_}) -> false; -vat_1({literal,_}=Lit, Required) -> vat_1(get_literal_type(Lit), Required); -vat_1(nil, list) -> true; -vat_1({tuple,SzA,EsA}, {tuple,SzB,EsB}) -> - if - is_list(SzB) -> - tuple_sz(SzA) >= tuple_sz(SzB) andalso vat_elements(EsA, EsB); - SzA =:= SzB -> - vat_elements(EsA, EsB); - SzA =/= SzB -> - false - end; -vat_1(_, _) -> false. - -vat_elements(EsA, EsB) -> - maps:fold(fun(Key, Req, Acc) -> - case EsA of - #{ Key := Given } -> Acc andalso vat_1(Given, Req); - #{} -> false - end - end, true, EsB). - allocate(Tag, Stk, Heap, Live, #vst{current=#st{numy=none}=St}=Vst0) -> verify_live(Live, Vst0), Vst1 = Vst0#vst{current=St#st{numy=Stk}}, @@ -1512,48 +1582,39 @@ assert_unique_map_keys([_,_|_]=Ls) -> %%% bsm_match_state() -> - #ms{}. + #t_bs_context{}. bsm_match_state(Slots) -> - #ms{slots=Slots}. - -bsm_validate_context(Reg, Vst) -> - _ = bsm_get_context(Reg, Vst), - ok. - -bsm_get_context({Kind,_}=Reg, Vst) when Kind =:= x; Kind =:= y-> - case get_movable_term_type(Reg, Vst) of - #ms{}=Ctx -> Ctx; - _ -> error({no_bsm_context,Reg}) - end; -bsm_get_context(Reg, _) -> - error({bad_source,Reg}). + #t_bs_context{slots=Slots}. bsm_save(Reg, {atom,start}, Vst) -> %% Save point refering to where the match started. %% It is always valid. But don't forget to validate the context register. - bsm_validate_context(Reg, Vst), + assert_type(#t_bs_context{}, Reg, Vst), Vst; bsm_save(Reg, SavePoint, Vst) -> - case bsm_get_context(Reg, Vst) of - #ms{valid=Bits,slots=Slots}=Ctxt0 when SavePoint < Slots -> - Ctx = Ctxt0#ms{valid=Bits bor (1 bsl SavePoint),slots=Slots}, - override_type(Ctx, Reg, Vst); - _ -> error({illegal_save,SavePoint}) + case get_movable_term_type(Reg, Vst) of + #t_bs_context{valid=Bits,slots=Slots}=Ctxt0 when SavePoint < Slots -> + Ctx = Ctxt0#t_bs_context{valid=Bits bor (1 bsl SavePoint), + slots=Slots}, + override_type(Ctx, Reg, Vst); + _ -> + error({illegal_save, SavePoint}) end. bsm_restore(Reg, {atom,start}, Vst) -> %% (Mostly) automatic save point refering to where the match started. %% It is always valid. But don't forget to validate the context register. - bsm_validate_context(Reg, Vst), + assert_type(#t_bs_context{}, Reg, Vst), Vst; bsm_restore(Reg, SavePoint, Vst) -> - case bsm_get_context(Reg, Vst) of - #ms{valid=Bits,slots=Slots} when SavePoint < Slots -> - case Bits band (1 bsl SavePoint) of - 0 -> error({illegal_restore,SavePoint,not_set}); - _ -> Vst - end; - _ -> error({illegal_restore,SavePoint,range}) + case get_movable_term_type(Reg, Vst) of + #t_bs_context{valid=Bits,slots=Slots} when SavePoint < Slots -> + case Bits band (1 bsl SavePoint) of + 0 -> error({illegal_restore, SavePoint, not_set}); + _ -> Vst + end; + _ -> + error({illegal_restore, SavePoint, range}) end. validate_select_val(_Fail, _Choices, _Src, #vst{current=none}=Vst) -> @@ -1569,7 +1630,7 @@ validate_select_val(Fail, [Val,{f,L}|T], Src, Vst0) -> update_ne_types(Src, Val, FailVst) end), validate_select_val(Fail, T, Src, Vst); -validate_select_val(Fail, [], _, Vst) -> +validate_select_val(Fail, [], _Src, Vst) -> branch(Fail, Vst, fun(SuccVst) -> %% The next instruction is never executed. @@ -1581,7 +1642,7 @@ validate_select_tuple_arity(_Fail, _Choices, _Src, #vst{current=none}=Vst) -> %% can't reach the fail label or any of the remaining choices. Vst; validate_select_tuple_arity(Fail, [Arity,{f,L}|T], Tuple, Vst0) -> - Type = {tuple, Arity, #{}}, + Type = #t_tuple{exact=true,size=Arity}, Vst = branch(L, Vst0, fun(BranchVst) -> update_type(fun meet/2, Type, Tuple, BranchVst) @@ -1597,63 +1658,94 @@ validate_select_tuple_arity(Fail, [], _, #vst{}=Vst) -> kill_state(SuccVst) end). -infer_types({Kind,_}=Reg, Vst) when Kind =:= x; Kind =:= y -> - infer_types(get_reg_vref(Reg, Vst), Vst); -infer_types(#value_ref{}=Ref, #vst{current=#st{vs=Vs}}) -> +%% +%% Infers types from comparisons, looking at the expressions that produced the +%% compared values and updates their types if we've learned something new from +%% the comparison. +%% + +infer_types(CompareOp, {Kind,_}=LHS, RHS, Vst) when Kind =:= x; Kind =:= y -> + infer_types(CompareOp, get_reg_vref(LHS, Vst), RHS, Vst); +infer_types(CompareOp, LHS, {Kind,_}=RHS, Vst) when Kind =:= x; Kind =:= y -> + infer_types(CompareOp, LHS, get_reg_vref(RHS, Vst), Vst); +infer_types(CompareOp, LHS, RHS, #vst{current=#st{vs=Vs}}=Vst0) -> case Vs of - #{ Ref := Entry } -> infer_types_1(Entry); - #{} -> fun(_, S) -> S end + #{ LHS := LEntry, RHS := REntry } -> + Vst = infer_types_1(LEntry, RHS, CompareOp, Vst0), + infer_types_1(REntry, LHS, CompareOp, Vst); + #{ LHS := LEntry } -> + infer_types_1(LEntry, RHS, CompareOp, Vst0); + #{ RHS := REntry } -> + infer_types_1(REntry, LHS, CompareOp, Vst0); + #{} -> + Vst0 + end. + +infer_types_1(#value{op={bif,'=:='},args=[LHS,RHS]}, Val, Op, Vst) -> + case Val of + {atom, Bool} when Op =:= eq_exact, Bool; Op =:= ne_exact, not Bool -> + update_eq_types(LHS, RHS, Vst); + {atom, Bool} when Op =:= ne_exact, Bool; Op =:= eq_exact, not Bool -> + update_ne_types(LHS, RHS, Vst); + _ -> + Vst end; -infer_types(_, #vst{}) -> - fun(_, S) -> S end. - -infer_types_1(#value{op={bif,'=:='},args=[LHS,RHS]}) -> - fun({atom,true}, S) -> - %% Either side might contain something worth inferring, so we need - %% to check them both. - Infer_L = infer_types(RHS, S), - Infer_R = infer_types(LHS, S), - Infer_R(RHS, Infer_L(LHS, S)); - (_, S) -> S +infer_types_1(#value{op={bif,'=/='},args=[LHS,RHS]}, Val, Op, Vst) -> + case Val of + {atom, Bool} when Op =:= ne_exact, Bool; Op =:= eq_exact, not Bool -> + update_ne_types(LHS, RHS, Vst); + {atom, Bool} when Op =:= eq_exact, Bool; Op =:= ne_exact, not Bool -> + update_eq_types(LHS, RHS, Vst); + _ -> + Vst end; -infer_types_1(#value{op={bif,element},args=[{integer,Index}=Key,Tuple]}) -> - fun(Val, S) -> - Type = {tuple,[Index], #{ Key => get_term_type(Val, S) }}, - update_type(fun meet/2, Type, Tuple, S) +infer_types_1(#value{op={bif,element},args=[{integer,Index},Tuple]}, + Val, Op, Vst) when Index >= 1 -> + ElementType = get_term_type(Val, Vst), + Es = beam_types:set_element_type(Index, ElementType, #{}), + Type = #t_tuple{size=Index,elements=Es}, + case Op of + eq_exact -> update_type(fun meet/2, Type, Tuple, Vst); + ne_exact -> update_type(fun subtract/2, Type, Tuple, Vst) end; -infer_types_1(#value{op={bif,is_atom},args=[Src]}) -> - infer_type_test_bif({atom,[]}, Src); -infer_types_1(#value{op={bif,is_boolean},args=[Src]}) -> - infer_type_test_bif(bool, Src); -infer_types_1(#value{op={bif,is_binary},args=[Src]}) -> - infer_type_test_bif(binary, Src); -infer_types_1(#value{op={bif,is_bitstring},args=[Src]}) -> - infer_type_test_bif(binary, Src); -infer_types_1(#value{op={bif,is_float},args=[Src]}) -> - infer_type_test_bif(float, Src); -infer_types_1(#value{op={bif,is_integer},args=[Src]}) -> - infer_type_test_bif({integer,{}}, Src); -infer_types_1(#value{op={bif,is_list},args=[Src]}) -> - infer_type_test_bif(list, Src); -infer_types_1(#value{op={bif,is_map},args=[Src]}) -> - infer_type_test_bif(map, Src); -infer_types_1(#value{op={bif,is_number},args=[Src]}) -> - infer_type_test_bif(number, Src); -infer_types_1(#value{op={bif,is_tuple},args=[Src]}) -> - infer_type_test_bif({tuple,[0],#{}}, Src); -infer_types_1(#value{op={bif,tuple_size}, args=[Tuple]}) -> - fun({integer,Arity}, S) -> - update_type(fun meet/2, {tuple,Arity,#{}}, Tuple, S); - (_, S) -> S +infer_types_1(#value{op={bif,is_atom},args=[Src]}, Val, Op, Vst) -> + infer_type_test_bif(#t_atom{}, Src, Val, Op, Vst); +infer_types_1(#value{op={bif,is_boolean},args=[Src]}, Val, Op, Vst) -> + infer_type_test_bif(beam_types:make_boolean(), Src, Val, Op, Vst); +infer_types_1(#value{op={bif,is_binary},args=[Src]}, Val, Op, Vst) -> + infer_type_test_bif(#t_bitstring{unit=8}, Src, Val, Op, Vst); +infer_types_1(#value{op={bif,is_bitstring},args=[Src]}, Val, Op, Vst) -> + infer_type_test_bif(#t_bitstring{}, Src, Val, Op, Vst); +infer_types_1(#value{op={bif,is_float},args=[Src]}, Val, Op, Vst) -> + infer_type_test_bif(float, Src, Val, Op, Vst); +infer_types_1(#value{op={bif,is_integer},args=[Src]}, Val, Op, Vst) -> + infer_type_test_bif(#t_integer{}, Src, Val, Op, Vst); +infer_types_1(#value{op={bif,is_list},args=[Src]}, Val, Op, Vst) -> + infer_type_test_bif(list, Src, Val, Op, Vst); +infer_types_1(#value{op={bif,is_map},args=[Src]}, Val, Op, Vst) -> + infer_type_test_bif(#t_map{}, Src, Val, Op, Vst); +infer_types_1(#value{op={bif,is_number},args=[Src]}, Val, Op, Vst) -> + infer_type_test_bif(number, Src, Val, Op, Vst); +infer_types_1(#value{op={bif,is_tuple},args=[Src]}, Val, Op, Vst) -> + infer_type_test_bif(#t_tuple{}, Src, Val, Op, Vst); +infer_types_1(#value{op={bif,tuple_size}, args=[Tuple]}, + {integer,Arity}, Op, Vst) -> + Type = #t_tuple{exact=true,size=Arity}, + case Op of + eq_exact -> update_type(fun meet/2, Type, Tuple, Vst); + ne_exact -> update_type(fun subtract/2, Type, Tuple, Vst) end; -infer_types_1(_) -> - fun(_, S) -> S end. - -infer_type_test_bif(Type, Src) -> - fun({atom,true}, S) -> - update_type(fun meet/2, Type, Src, S); - (_, S) -> - S +infer_types_1(_, _, _, Vst) -> + Vst. + +infer_type_test_bif(Type, Src, Val, Op, Vst) -> + case Val of + {atom, Bool} when Op =:= eq_exact, Bool; Op =:= ne_exact, not Bool -> + update_type(fun meet/2, Type, Src, Vst); + {atom, Bool} when Op =:= ne_exact, Bool; Op =:= eq_exact, not Bool -> + update_type(fun subtract/2, Type, Src, Vst); + _ -> + Vst end. %%% @@ -1764,43 +1856,58 @@ update_type(Merge, With, #value_ref{}=Ref, Vst) -> update_type(Merge, With, {Kind,_}=Reg, Vst) when Kind =:= x; Kind =:= y -> update_type(Merge, With, get_reg_vref(Reg, Vst), Vst); update_type(Merge, With, Literal, Vst) -> - assert_literal(Literal), %% Literals always retain their type, but we still need to bail on type %% conflicts. - case Merge(Literal, With) of - none -> throw({type_conflict, Literal, With}); + Type = get_literal_type(Literal), + case Merge(Type, With) of + none -> throw({type_conflict, Type, With}); _Type -> Vst end. -update_ne_types(LHS, RHS, Vst) -> +update_eq_types(LHS, RHS, Vst0) -> + LType = get_term_type(LHS, Vst0), + RType = get_term_type(RHS, Vst0), + + Vst1 = update_type(fun meet/2, RType, LHS, Vst0), + Vst = update_type(fun meet/2, LType, RHS, Vst1), + + infer_types(eq_exact, LHS, RHS, Vst). + +update_ne_types(LHS, RHS, Vst0) -> + Vst1 = update_ne_types_1(LHS, RHS, Vst0), + Vst = update_ne_types_1(RHS, LHS, Vst1), + + infer_types(ne_exact, LHS, RHS, Vst). + +update_ne_types_1(LHS, RHS, Vst0) -> %% While updating types on equality is fairly straightforward, inequality %% is a bit trickier since all we know is that the *value* of LHS differs %% from RHS, so we can't blindly subtract their types. %% - %% Consider `number =/= {integer,[]}`; all we know is that LHS isn't equal + %% Consider `number =/= #t_integer{}`; all we know is that LHS isn't equal %% to some *specific integer* of unknown value, and if we were to subtract - %% {integer,[]} we would erroneously infer that the new type is {float,[]}. + %% #t_integer{} we would erroneously infer that the new type is float. %% %% Therefore, we only subtract when we know that RHS has a specific value. - RType = get_term_type(RHS, Vst), - case is_literal(RType) of - true -> update_type(fun subtract/2, RType, LHS, Vst); - false -> Vst + RType = get_term_type(RHS, Vst0), + case beam_types:is_singleton_type(RType) of + true -> + Vst = update_type(fun subtract/2, RType, LHS, Vst0), + + %% If LHS has a specific value after subtraction we can infer types + %% as if we've made an exact match, which is much stronger than + %% ne_exact. + LType = get_term_type(LHS, Vst), + case beam_types:get_singleton_value(LType) of + {ok, Value} -> + infer_types(eq_exact, LHS, value_to_literal(Value), Vst); + error -> + Vst + end; + false -> + Vst0 end. -update_eq_types(LHS, RHS, Vst0) -> - %% Either side might contain something worth inferring, so we need - %% to check them both. - Infer_L = infer_types(RHS, Vst0), - Infer_R = infer_types(LHS, Vst0), - Vst1 = Infer_R(RHS, Infer_L(LHS, Vst0)), - - T1 = get_term_type(LHS, Vst1), - T2 = get_term_type(RHS, Vst1), - - Vst = update_type(fun meet/2, T2, LHS, Vst1), - update_type(fun meet/2, T1, RHS, Vst). - %% Helper functions for the above. assign_1(Src, Dst, Vst0) -> @@ -1851,16 +1958,9 @@ get_reg_vref({y,_}=Src, #vst{current=#st{ys=Ys}}) -> end. set_type(Type, #value_ref{}=Ref, #vst{current=#st{vs=Vs0}=St}=Vst) -> - case Vs0 of - #{ Ref := #value{}=Entry } -> - Vs = Vs0#{ Ref => Entry#value{type=Type} }, - Vst#vst{current=St#st{vs=Vs}}; - #{} -> - %% Dead references may happen during type inference and are not an - %% error in and of themselves. If a problem were to arise from this - %% it'll explode elsewhere. - Vst - end. + #{ Ref := #value{}=Entry } = Vs0, + Vs = Vs0#{ Ref => Entry#value{type=Type} }, + Vst#vst{current=St#st{vs=Vs}}. new_value(Type, Op, Ss, #vst{current=#st{vs=Vs0}=St,ref_ctr=Counter}=Vst) -> Ref = #value_ref{id=Counter}, @@ -1868,9 +1968,9 @@ new_value(Type, Op, Ss, #vst{current=#st{vs=Vs0}=St,ref_ctr=Counter}=Vst) -> {Ref, Vst#vst{current=St#st{vs=Vs},ref_ctr=Counter+1}}. -kill_catch_tag(Reg, #vst{current=#st{ct=[Fail|Fails]}=St}=Vst0) -> - Vst = Vst0#vst{current=St#st{ct=Fails,fls=undefined}}, - {_, Fail} = get_tag_type(Reg, Vst), %Assertion. +kill_catch_tag(Reg, #vst{current=#st{ct=[Tag|Tags]}=St}=Vst0) -> + Vst = Vst0#vst{current=St#st{ct=Tags,fls=undefined}}, + Tag = get_tag_type(Reg, Vst), %Assertion. kill_tag(Reg, Vst). check_try_catch_tags(Type, {y,N}=Reg, Vst) -> @@ -1915,308 +2015,45 @@ is_literal({integer,I}) when is_integer(I) -> true; is_literal({literal,_L}) -> true; is_literal(_) -> false. -%% The possible types. -%% -%% First non-term types: -%% -%% initialized Only for Y registers. Means that the Y register -%% has been initialized with some valid term so that -%% it is safe to pass to the garbage collector. -%% NOT safe to use in any other way (will not crash the -%% emulator, but clearly points to a bug in the compiler). -%% -%% {catchtag,[Lbl]} A special term used within a catch. Must only be used -%% by the catch instructions; NOT safe to use in other -%% instructions. -%% -%% {trytag,[Lbl]} A special term used within a try block. Must only be -%% used by the catch instructions; NOT safe to use in other -%% instructions. -%% -%% exception Can only be used as a type returned by -%% call_return_type/2 (which gives the type of the value -%% returned by a call). Thus 'exception' is never stored -%% as type descriptor for a register. -%% -%% #ms{} A match context for bit syntax matching. We do allow -%% it to moved/to from stack, but otherwise it must only -%% be accessed by bit syntax matching instructions. -%% -%% -%% Normal terms: -%% -%% term Any valid Erlang (but not of the special types above). -%% -%% binary Binary or bitstring. -%% -%% bool The atom 'true' or the atom 'false'. -%% -%% cons Cons cell: [_|_] -%% -%% nil Empty list: [] -%% -%% list List: [] or [_|_] -%% -%% {tuple,[Sz],Es} Tuple. An element has been accessed using -%% element/2 or setelement/3 so that it is known that -%% the type is a tuple of size at least Sz. Es is a map -%% containing known types by tuple index. -%% -%% {tuple,Sz,Es} Tuple. A test_arity instruction has been seen -%% so that it is known that the size is exactly Sz. -%% -%% {atom,[]} Atom. -%% {atom,Atom} -%% -%% {integer,[]} Integer. -%% {integer,Integer} -%% -%% {float,[]} Float. -%% {float,Float} -%% -%% number Integer or Float of unknown value -%% -%% map Map. -%% -%% none A conflict in types. There will be an exception at runtime. -%% - -%% join(Type1, Type2) -> Type -%% Return the most specific type possible. -join(Same, Same) -> - Same; -join(none, Other) -> - Other; -join(Other, none) -> - Other; -join({literal,_}=T1, T2) -> - join_literal(T1, T2); -join(T1, {literal,_}=T2) -> - join_literal(T2, T1); -join({tuple,Size,EsA}, {tuple,Size,EsB}) -> - Es = join_tuple_elements(tuple_sz(Size), EsA, EsB), - {tuple, Size, Es}; -join({tuple,A,EsA}, {tuple,B,EsB}) -> - Size = min(tuple_sz(A), tuple_sz(B)), - Es = join_tuple_elements(Size, EsA, EsB), - {tuple, [Size], Es}; -join({Type,A}, {Type,B}) - when Type =:= atom; Type =:= integer; Type =:= float -> - if A =:= B -> {Type,A}; - true -> {Type,[]} - end; -join({Type,_}, number) - when Type =:= integer; Type =:= float -> - number; -join(number, {Type,_}) - when Type =:= integer; Type =:= float -> - number; -join({integer,_}, {float,_}) -> - number; -join({float,_}, {integer,_}) -> - number; -join(bool, {atom,A}) -> - join_bool(A); -join({atom,A}, bool) -> - join_bool(A); -join({atom,A}, {atom,B}) when is_boolean(A), is_boolean(B) -> - bool; -join({atom,_}, {atom,_}) -> - {atom,[]}; -join(#ms{id=Id1,valid=B1,slots=Slots1}, - #ms{id=Id2,valid=B2,slots=Slots2}) -> - Id = if - Id1 =:= Id2 -> Id1; - true -> make_ref() - end, - #ms{id=Id,valid=B1 band B2,slots=min(Slots1, Slots2)}; -join(T1, T2) when T1 =/= T2 -> - %% We've exhaused all other options, so the type must either be a list or - %% a 'term'. - join_list(T1, T2). - -join_tuple_elements(Limit, EsA, EsB) -> - Es0 = join_elements(EsA, EsB), - maps:filter(fun({integer,Index}, _Type) -> Index =< Limit end, Es0). +%% `dialyzer` complains about the float and general literal cases never being +%% matched and I don't like suppressing warnings. Should they become possible +%% I'm sure `dialyzer` will warn about it. +value_to_literal([]) -> nil; +value_to_literal(A) when is_atom(A) -> {atom,A}; +value_to_literal(I) when is_integer(I) -> {integer,I}. -join_elements(Es1, Es2) -> - Keys = if - map_size(Es1) =< map_size(Es2) -> maps:keys(Es1); - map_size(Es1) > map_size(Es2) -> maps:keys(Es2) - end, - join_elements_1(Keys, Es1, Es2, #{}). - -join_elements_1([Key | Keys], Es1, Es2, Acc0) -> - Type = case {Es1, Es2} of - {#{ Key := Same }, #{ Key := Same }} -> Same; - {#{ Key := Type1 }, #{ Key := Type2 }} -> join(Type1, Type2); - {#{}, #{}} -> term - end, - Acc = set_element_type(Key, Type, Acc0), - join_elements_1(Keys, Es1, Es2, Acc); -join_elements_1([], _Es1, _Es2, Acc) -> - Acc. - -%% Joins types of literals; note that the left argument must either be a -%% literal or exactly equal to the second argument. -join_literal(Same, Same) -> - Same; -join_literal({literal,_}=Lit, T) -> - join_literal(T, get_literal_type(Lit)); -join_literal(T1, T2) -> - %% We're done extracting the types, try merging them again. - join(T1, T2). - -join_list(nil, cons) -> list; -join_list(nil, list) -> list; -join_list(cons, list) -> list; -join_list(T, nil) -> join_list(nil, T); -join_list(T, cons) -> join_list(cons, T); -join_list(_, _) -> - %% Not a list, so it must be a term. - term. - -join_bool([]) -> {atom,[]}; -join_bool(true) -> bool; -join_bool(false) -> bool; -join_bool(_) -> {atom,[]}. - -%% meet(Type1, Type2) -> Type -%% Return the meet of two types. The meet is a more specific type. -%% It will be 'none' if the types are in conflict. - -meet(Same, Same) -> - Same; -meet(term, Other) -> - Other; -meet(Other, term) -> - Other; -meet(#ms{}, binary) -> - #ms{}; -meet(binary, #ms{}) -> - #ms{}; -meet({literal,_}, {literal,_}) -> - none; -meet(T1, {literal,_}=T2) -> - meet(T2, T1); -meet({literal,_}=T1, T2) -> - case meet(get_literal_type(T1), T2) of - none -> none; - _ -> T1 - end; -meet(T1, T2) -> - case {erlang:min(T1, T2),erlang:max(T1, T2)} of - {{atom,_}=A,{atom,[]}} -> A; - {bool,{atom,B}=Atom} when is_boolean(B) -> Atom; - {bool,{atom,[]}} -> bool; - {cons,list} -> cons; - {{float,_}=T,{float,[]}} -> T; - {{integer,_}=T,{integer,[]}} -> T; - {list,nil} -> nil; - {number,{integer,_}=T} -> T; - {number,{float,_}=T} -> T; - {{tuple,Size1,Es1},{tuple,Size2,Es2}} -> - Es = meet_elements(Es1, Es2), - case {Size1,Size2,Es} of - {_, _, none} -> - none; - {[Sz1],[Sz2],_} -> - Sz = erlang:max(Sz1, Sz2), - assert_tuple_elements(Sz, Es), - {tuple,[Sz],Es}; - {Sz1,[Sz2],_} when Sz2 =< Sz1 -> - assert_tuple_elements(Sz1, Es), - {tuple,Sz1,Es}; - {Sz,Sz,_} -> - assert_tuple_elements(Sz, Es), - {tuple,Sz,Es}; - {_,_,_} -> - none - end; - {_,_} -> none +%% These are just wrappers around their equivalents in beam_types, which +%% handle the validator-specific #t_abstract{} type. +%% +%% The funny-looking abstract types produced here are intended to provoke +%% errors on actual use; they do no harm just lying around. + +normalize(#t_abstract{}=A) -> error({abstract_type, A}); +normalize(T) -> beam_types:normalize(T). + +join(Same, Same) -> Same; +join(#t_abstract{}=A, B) -> #t_abstract{kind={join, A, B}}; +join(A, #t_abstract{}=B) -> #t_abstract{kind={join, A, B}}; +join(A, B) -> beam_types:join(A, B). + +meet(Same, Same) -> Same; +meet(#t_abstract{}=A, B) -> #t_abstract{kind={meet, A, B}}; +meet(A, #t_abstract{}=B) -> #t_abstract{kind={meet, A, B}}; +meet(A, B) -> beam_types:meet(A, B). + +subtract(#t_abstract{}=A, B) -> #t_abstract{kind={subtract, A, B}}; +subtract(A, #t_abstract{}=B) -> #t_abstract{kind={subtract, A, B}}; +subtract(A, B) -> beam_types:subtract(A, B). + +assert_type(RequiredType, Term, Vst) -> + GivenType = get_movable_term_type(Term, Vst), + case meet(RequiredType, GivenType) of + GivenType -> + ok; + _RequiredType -> + error({bad_type,{needed,RequiredType},{actual,GivenType}}) end. -meet_elements(Es1, Es2) -> - Keys = maps:keys(Es1) ++ maps:keys(Es2), - meet_elements_1(Keys, Es1, Es2, #{}). - -meet_elements_1([Key | Keys], Es1, Es2, Acc) -> - case {Es1, Es2} of - {#{ Key := Type1 }, #{ Key := Type2 }} -> - case meet(Type1, Type2) of - none -> none; - Type -> meet_elements_1(Keys, Es1, Es2, Acc#{ Key => Type }) - end; - {#{ Key := Type1 }, _} -> - meet_elements_1(Keys, Es1, Es2, Acc#{ Key => Type1 }); - {_, #{ Key := Type2 }} -> - meet_elements_1(Keys, Es1, Es2, Acc#{ Key => Type2 }) - end; -meet_elements_1([], _Es1, _Es2, Acc) -> - Acc. - -%% No tuple elements may have an index above the known size. -assert_tuple_elements(Limit, Es) -> - true = maps:fold(fun({integer,Index}, _T, true) -> - Index =< Limit - end, true, Es). %Assertion. - -%% subtract(Type1, Type2) -> Type -%% Subtract Type2 from Type2. Example: -%% subtract(list, nil) -> cons - -subtract(Same, Same) -> none; -subtract(list, nil) -> cons; -subtract(list, cons) -> nil; -subtract(number, {integer,[]}) -> {float,[]}; -subtract(number, {float,[]}) -> {integer,[]}; -subtract(bool, {atom,false}) -> {atom, true}; -subtract(bool, {atom,true}) -> {atom, false}; -subtract(Type, _) -> Type. - -assert_type(WantedType, Term, Vst) -> - Type = get_term_type(Term, Vst), - assert_type(WantedType, Type). - -assert_type(Correct, Correct) -> ok; -assert_type(float, {float,_}) -> ok; -assert_type(tuple, {tuple,_,_}) -> ok; -assert_type(tuple, {literal,Tuple}) when is_tuple(Tuple) -> ok; -assert_type({tuple_element,I}, {tuple,[Sz],_}) - when 1 =< I, I =< Sz -> - ok; -assert_type({tuple_element,I}, {tuple,Sz,_}) - when is_integer(Sz), 1 =< I, I =< Sz -> - ok; -assert_type({tuple_element,I}, {literal,Lit}) when I =< tuple_size(Lit) -> - ok; -assert_type(cons, {literal,[_|_]}) -> - ok; -assert_type(Needed, Actual) -> - error({bad_type,{needed,Needed},{actual,Actual}}). - -get_element_type(Key, Src, Vst) -> - get_element_type_1(Key, get_term_type(Src, Vst)). - -get_element_type_1({integer,_}=Key, {tuple,_Sz,Es}) -> - case Es of - #{ Key := Type } -> Type; - #{} -> term - end; -get_element_type_1(_Index, _Type) -> - term. - -set_element_type(_Key, none, Es) -> - Es; -set_element_type(Key, term, Es) -> - maps:remove(Key, Es); -set_element_type(Key, Type, Es) -> - Es#{ Key => Type }. - -get_tuple_size({integer,[]}) -> 0; -get_tuple_size({integer,Sz}) -> Sz; -get_tuple_size(_) -> 0. - validate_src(Ss, Vst) when is_list(Ss) -> _ = [assert_term(S, Vst) || S <- Ss], ok. @@ -2227,7 +2064,8 @@ validate_src(Ss, Vst) when is_list(Ss) -> get_term_type(Src, Vst) -> case get_movable_term_type(Src, Vst) of - #ms{} -> error({match_context,Src}); + #t_bs_context{} -> error({match_context,Src}); + #t_abstract{} -> error({abstract_term,Src}); Type -> Type end. @@ -2237,12 +2075,11 @@ get_term_type(Src, Vst) -> get_movable_term_type(Src, Vst) -> case get_raw_type(Src, Vst) of + #t_abstract{kind=unfinished_tuple=Kind} -> error({Kind,Src}); initialized -> error({unassigned,Src}); uninitialized -> error({uninitialized_reg,Src}); {catchtag,_} -> error({catchtag,Src}); {trytag,_} -> error({trytag,Src}); - tuple_in_progress -> error({tuple_in_progress,Src}); - {literal,_}=Lit -> get_literal_type(Lit); Type -> Type end. @@ -2284,30 +2121,18 @@ get_raw_type(#value_ref{}=Ref, #vst{current=#st{vs=Vs}}) -> get_raw_type(Src, #vst{}) -> get_literal_type(Src). -get_literal_type(nil=T) -> T; -get_literal_type({atom,A}=T) when is_atom(A) -> T; -get_literal_type({float,F}=T) when is_float(F) -> T; -get_literal_type({integer,I}=T) when is_integer(I) -> T; -get_literal_type({literal,[_|_]}) -> cons; -get_literal_type({literal,Bitstring}) when is_bitstring(Bitstring) -> binary; -get_literal_type({literal,Map}) when is_map(Map) -> map; -get_literal_type({literal,Tuple}) when is_tuple(Tuple) -> glt_1(Tuple); -get_literal_type({literal,_}) -> term; -get_literal_type(T) -> error({not_literal,T}). - -glt_1([]) -> nil; -glt_1(A) when is_atom(A) -> {atom, A}; -glt_1(F) when is_float(F) -> {float, F}; -glt_1(I) when is_integer(I) -> {integer, I}; -glt_1(T) when is_tuple(T) -> - {Es,_} = foldl(fun(Val, {Es0, Index}) -> - Type = glt_1(Val), - Es = set_element_type({integer,Index}, Type, Es0), - {Es, Index + 1} - end, {#{}, 1}, tuple_to_list(T)), - {tuple, tuple_size(T), Es}; -glt_1(L) -> - {literal, L}. +get_literal_type(nil) -> + beam_types:make_type_from_value([]); +get_literal_type({atom,A}) when is_atom(A) -> + beam_types:make_type_from_value(A); +get_literal_type({float,F}) when is_float(F) -> + beam_types:make_type_from_value(F); +get_literal_type({integer,I}) when is_integer(I) -> + beam_types:make_type_from_value(I); +get_literal_type({literal,L}) -> + beam_types:make_type_from_value(L); +get_literal_type(T) -> + error({not_literal,T}). %%% %%% Branch tracking @@ -2325,10 +2150,12 @@ glt_1(L) -> SuccFun :: BranchFun) -> #vst{} when BranchFun :: fun((#vst{}) -> #vst{}). branch(Lbl, Vst0, FailFun, SuccFun) -> + validate_branch(Lbl, Vst0), #vst{current=St0} = Vst0, + try FailFun(Vst0) of Vst1 -> - Vst2 = branch_state(Lbl, Vst1), + Vst2 = fork_state(Lbl, Vst1), Vst = Vst2#vst{current=St0}, try SuccFun(Vst) of V -> V @@ -2346,6 +2173,24 @@ branch(Lbl, Vst0, FailFun, SuccFun) -> SuccFun(Vst0) end. +validate_branch(Lbl, #vst{current=#st{ct=Tags}}) -> + validate_branch_1(Lbl, Tags). + +validate_branch_1(Lbl, [{trytag, FailLbls} | Tags]) -> + %% 'try_case' assumes that an exception has been thrown, so a direct branch + %% will crash the emulator. + %% + %% (Jumping to a 'catch_end' is fine however as it will simply nop in the + %% absence of an exception.) + case ordsets:is_element(Lbl, FailLbls) of + true -> error({illegal_branch, try_handler, Lbl}); + false -> validate_branch_1(Lbl, Tags) + end; +validate_branch_1(Lbl, [_ | Tags]) -> + validate_branch_1(Lbl, Tags); +validate_branch_1(_Lbl, []) -> + ok. + %% A shorthand version of branch/4 for when the state is only altered on %% success. branch(Fail, Vst, SuccFun) -> @@ -2353,12 +2198,12 @@ branch(Fail, Vst, SuccFun) -> %% Directly branches off the state. This is an "internal" operation that should %% be used sparingly. -branch_state(0, #vst{}=Vst) -> +fork_state(0, #vst{}=Vst) -> %% If the instruction fails, the stack may be scanned looking for a catch %% tag. Therefore the Y registers must be initialized at this point. verify_y_init(Vst), Vst; -branch_state(L, #vst{current=St,branched=B,ref_ctr=Counter0}=Vst) -> +fork_state(L, #vst{current=St,branched=B,ref_ctr=Counter0}=Vst) -> case gb_trees:is_defined(L, B) of true -> {MergedSt, Counter} = merge_states(L, St, B, Counter0), @@ -2438,10 +2283,10 @@ merge_tags(uninitialized, _) -> uninitialized; merge_tags(_, uninitialized) -> uninitialized; -merge_tags({catchtag,T0}, {catchtag,T1}) -> - {catchtag, ordsets:from_list(T0 ++ T1)}; -merge_tags({trytag,T0}, {trytag,T1}) -> - {trytag, ordsets:from_list(T0 ++ T1)}; +merge_tags({trytag, LblsA}, {trytag, LblsB}) -> + {trytag, ordsets:union(LblsA, LblsB)}; +merge_tags({catchtag, LblsA}, {catchtag, LblsB}) -> + {catchtag, ordsets:union(LblsA, LblsB)}; merge_tags(_A, _B) -> %% All other combinations leave the register initialized. Errors arising %% from this will be caught later on. @@ -2512,13 +2357,14 @@ merge_stk(_, _) -> undecided. merge_ct(S, S) -> S; merge_ct(Ct0, Ct1) -> merge_ct_1(Ct0, Ct1). -merge_ct_1([C0|Ct0], [C1|Ct1]) -> - [ordsets:from_list(C0++C1)|merge_ct_1(Ct0, Ct1)]; -merge_ct_1([], []) -> []; -merge_ct_1(_, _) -> undecided. - -tuple_sz([Sz]) -> Sz; -tuple_sz(Sz) -> Sz. +merge_ct_1([], []) -> + []; +merge_ct_1([{trytag, LblsA} | CtA], [{trytag, LblsB} | CtB]) -> + [{trytag, ordsets:union(LblsA, LblsB)} | merge_ct_1(CtA, CtB)]; +merge_ct_1([{catchtag, LblsA} | CtA], [{catchtag, LblsB} | CtB]) -> + [{catchtag, ordsets:union(LblsA, LblsB)} | merge_ct_1(CtA, CtB)]; +merge_ct_1(_, _) -> + undecided. verify_y_init(#vst{current=#st{numy=NumY,ys=Ys}}=Vst) when is_integer(NumY) -> HighestY = maps:fold(fun({y,Y}, _, Acc) -> max(Y, Acc) end, -1, Ys), @@ -2661,319 +2507,46 @@ assert_not_fragile(Lit, #vst{}) -> ok. %%% -%%% Return/argument types of BIFs -%%% - -bif_return_type('-', Src, Vst) -> - arith_return_type(Src, Vst); -bif_return_type('+', Src, Vst) -> - arith_return_type(Src, Vst); -bif_return_type('*', Src, Vst) -> - arith_return_type(Src, Vst); -bif_return_type(abs, [Num], Vst) -> - case get_term_type(Num, Vst) of - {float,_}=T -> T; - {integer,_}=T -> T; - _ -> number - end; -bif_return_type(float, _, _) -> {float,[]}; -bif_return_type('/', _, _) -> {float,[]}; -%% Binary operations -bif_return_type('binary_part', [_,_], _) -> binary; -bif_return_type('binary_part', [_,_,_], _) -> binary; -bif_return_type('bit_size', [_], _) -> {integer,[]}; -bif_return_type('byte_size', [_], _) -> {integer,[]}; -%% Integer operations. -bif_return_type(ceil, [_], _) -> {integer,[]}; -bif_return_type('div', [_,_], _) -> {integer,[]}; -bif_return_type(floor, [_], _) -> {integer,[]}; -bif_return_type('rem', [_,_], _) -> {integer,[]}; -bif_return_type(length, [_], _) -> {integer,[]}; -bif_return_type(size, [_], _) -> {integer,[]}; -bif_return_type(trunc, [_], _) -> {integer,[]}; -bif_return_type(round, [_], _) -> {integer,[]}; -bif_return_type('band', [_,_], _) -> {integer,[]}; -bif_return_type('bor', [_,_], _) -> {integer,[]}; -bif_return_type('bxor', [_,_], _) -> {integer,[]}; -bif_return_type('bnot', [_], _) -> {integer,[]}; -bif_return_type('bsl', [_,_], _) -> {integer,[]}; -bif_return_type('bsr', [_,_], _) -> {integer,[]}; -%% Booleans. -bif_return_type('==', [_,_], _) -> bool; -bif_return_type('/=', [_,_], _) -> bool; -bif_return_type('=<', [_,_], _) -> bool; -bif_return_type('<', [_,_], _) -> bool; -bif_return_type('>=', [_,_], _) -> bool; -bif_return_type('>', [_,_], _) -> bool; -bif_return_type('=:=', [_,_], _) -> bool; -bif_return_type('=/=', [_,_], _) -> bool; -bif_return_type('not', [_], _) -> bool; -bif_return_type('and', [_,_], _) -> bool; -bif_return_type('or', [_,_], _) -> bool; -bif_return_type('xor', [_,_], _) -> bool; -bif_return_type(is_atom, [_], _) -> bool; -bif_return_type(is_boolean, [_], _) -> bool; -bif_return_type(is_binary, [_], _) -> bool; -bif_return_type(is_float, [_], _) -> bool; -bif_return_type(is_function, [_], _) -> bool; -bif_return_type(is_function, [_,_], _) -> bool; -bif_return_type(is_integer, [_], _) -> bool; -bif_return_type(is_list, [_], _) -> bool; -bif_return_type(is_map, [_], _) -> bool; -bif_return_type(is_number, [_], _) -> bool; -bif_return_type(is_pid, [_], _) -> bool; -bif_return_type(is_port, [_], _) -> bool; -bif_return_type(is_reference, [_], _) -> bool; -bif_return_type(is_tuple, [_], _) -> bool; -%% Misc. -bif_return_type(tuple_size, [_], _) -> {integer,[]}; -bif_return_type(map_size, [_], _) -> {integer,[]}; -bif_return_type(node, [], _) -> {atom,[]}; -bif_return_type(node, [_], _) -> {atom,[]}; -bif_return_type(hd, [_], _) -> term; -bif_return_type(tl, [_], _) -> term; -bif_return_type(get, [_], _) -> term; -bif_return_type(Bif, _, _) when is_atom(Bif) -> term. - -%% Generic -bif_arg_types(tuple_size, [_]) -> [{tuple,[0],#{}}]; -bif_arg_types(map_size, [_]) -> [map]; -bif_arg_types(is_map_key, [_,_]) -> [term, map]; -bif_arg_types(map_get, [_,_]) -> [term, map]; -bif_arg_types(length, [_]) -> [list]; -bif_arg_types(hd, [_]) -> [cons]; -bif_arg_types(tl, [_]) -> [cons]; -%% Boolean -bif_arg_types('not', [_]) -> [bool]; -bif_arg_types('and', [_,_]) -> [bool, bool]; -bif_arg_types('or', [_,_]) -> [bool, bool]; -bif_arg_types('xor', [_,_]) -> [bool, bool]; -%% Binary -bif_arg_types('binary_part', [_,_]) -> - PosLen = {tuple, 2, #{ {integer,1} => {integer,[]}, - {integer,2} => {integer,[]} }}, - [binary, PosLen]; -bif_arg_types('binary_part', [_,_,_]) -> - [binary, {integer,[]}, {integer,[]}]; -bif_arg_types('bit_size', [_]) -> [binary]; -bif_arg_types('byte_size', [_]) -> [binary]; -%% Numerical -bif_arg_types('-', [_]) -> [number]; -bif_arg_types('-', [_,_]) -> [number,number]; -bif_arg_types('+', [_]) -> [number]; -bif_arg_types('+', [_,_]) -> [number,number]; -bif_arg_types('*', [_,_]) -> [number, number]; -bif_arg_types('/', [_,_]) -> [number, number]; -bif_arg_types(abs, [_]) -> [number]; -bif_arg_types(ceil, [_]) -> [number]; -bif_arg_types(float, [_]) -> [number]; -bif_arg_types(floor, [_]) -> [number]; -bif_arg_types(trunc, [_]) -> [number]; -bif_arg_types(round, [_]) -> [number]; -%% Integer-specific -bif_arg_types('div', [_,_]) -> [{integer,[]}, {integer,[]}]; -bif_arg_types('rem', [_,_]) -> [{integer,[]}, {integer,[]}]; -bif_arg_types('band', [_,_]) -> [{integer,[]}, {integer,[]}]; -bif_arg_types('bor', [_,_]) -> [{integer,[]}, {integer,[]}]; -bif_arg_types('bxor', [_,_]) -> [{integer,[]}, {integer,[]}]; -bif_arg_types('bnot', [_]) -> [{integer,[]}]; -bif_arg_types('bsl', [_,_]) -> [{integer,[]}, {integer,[]}]; -bif_arg_types('bsr', [_,_]) -> [{integer,[]}, {integer,[]}]; -%% Unsafe type tests that may fail if an argument doesn't have the right type. -bif_arg_types(is_function, [_,_]) -> [term, {integer,[]}]; -bif_arg_types(_, Args) -> [term || _Arg <- Args]. - -is_bif_safe('/=', 2) -> true; -is_bif_safe('<', 2) -> true; -is_bif_safe('=/=', 2) -> true; -is_bif_safe('=:=', 2) -> true; -is_bif_safe('=<', 2) -> true; -is_bif_safe('==', 2) -> true; -is_bif_safe('>', 2) -> true; -is_bif_safe('>=', 2) -> true; -is_bif_safe(is_atom, 1) -> true; -is_bif_safe(is_boolean, 1) -> true; -is_bif_safe(is_binary, 1) -> true; -is_bif_safe(is_bitstring, 1) -> true; -is_bif_safe(is_float, 1) -> true; -is_bif_safe(is_function, 1) -> true; -is_bif_safe(is_integer, 1) -> true; -is_bif_safe(is_list, 1) -> true; -is_bif_safe(is_map, 1) -> true; -is_bif_safe(is_number, 1) -> true; -is_bif_safe(is_pid, 1) -> true; -is_bif_safe(is_port, 1) -> true; -is_bif_safe(is_reference, 1) -> true; -is_bif_safe(is_tuple, 1) -> true; -is_bif_safe(get, 1) -> true; -is_bif_safe(self, 0) -> true; -is_bif_safe(node, 0) -> true; -is_bif_safe(_, _) -> false. - -arith_return_type([A], Vst) -> - %% Unary '+' or '-'. - case get_term_type(A, Vst) of - {integer,_} -> {integer,[]}; - {float,_} -> {float,[]}; - _ -> number - end; -arith_return_type([A,B], Vst) -> - TypeA = get_term_type(A, Vst), - TypeB = get_term_type(B, Vst), - case {TypeA, TypeB} of - {{integer,_},{integer,_}} -> {integer,[]}; - {{float,_},_} -> {float,[]}; - {_,{float,_}} -> {float,[]}; - {_,_} -> number - end; -arith_return_type(_, _) -> number. - -%%% -%%% Return/argument types of calls +%%% Return/argument types of calls and BIFs %%% -call_return_type({extfunc,M,F,A}, Vst) -> call_return_type_1(M, F, A, Vst); -call_return_type(_, _) -> term. - -call_return_type_1(erlang, setelement, 3, Vst) -> - IndexType = get_term_type({x,0}, Vst), - TupleType = - case get_term_type({x,1}, Vst) of - {literal,Tuple}=Lit when is_tuple(Tuple) -> get_literal_type(Lit); - {tuple,_,_}=TT -> TT; - _ -> {tuple,[0],#{}} - end, - case IndexType of - {integer,I} when is_integer(I) -> - case meet({tuple,[I],#{}}, TupleType) of - {tuple, Sz, Es0} -> - ValueType = get_term_type({x,2}, Vst), - Es = set_element_type({integer,I}, ValueType, Es0), - {tuple, Sz, Es}; - none -> - TupleType - end; - _ -> - %% The index could point anywhere, so we must discard all element - %% information. - setelement(3, TupleType, #{}) - end; -call_return_type_1(erlang, '++', 2, Vst) -> - LType = get_term_type({x,0}, Vst), - RType = get_term_type({x,1}, Vst), - case LType =:= cons orelse RType =:= cons of - true -> - cons; - false -> - %% `[] ++ RHS` yields RHS, even if RHS is not a list - join(list, RType) - end; -call_return_type_1(erlang, '--', 2, _Vst) -> - list; -call_return_type_1(erlang, F, A, _) -> - erlang_mod_return_type(F, A); -call_return_type_1(lists, F, A, Vst) -> - lists_mod_return_type(F, A, Vst); -call_return_type_1(math, F, A, _) -> - math_mod_return_type(F, A); -call_return_type_1(M, F, A, _) when is_atom(M), is_atom(F), is_integer(A), A >= 0 -> - term. - -erlang_mod_return_type(exit, 1) -> exception; -erlang_mod_return_type(throw, 1) -> exception; -erlang_mod_return_type(error, 1) -> exception; -erlang_mod_return_type(error, 2) -> exception; -erlang_mod_return_type(F, A) when is_atom(F), is_integer(A), A >= 0 -> term. - -math_mod_return_type(cos, 1) -> {float,[]}; -math_mod_return_type(cosh, 1) -> {float,[]}; -math_mod_return_type(sin, 1) -> {float,[]}; -math_mod_return_type(sinh, 1) -> {float,[]}; -math_mod_return_type(tan, 1) -> {float,[]}; -math_mod_return_type(tanh, 1) -> {float,[]}; -math_mod_return_type(acos, 1) -> {float,[]}; -math_mod_return_type(acosh, 1) -> {float,[]}; -math_mod_return_type(asin, 1) -> {float,[]}; -math_mod_return_type(asinh, 1) -> {float,[]}; -math_mod_return_type(atan, 1) -> {float,[]}; -math_mod_return_type(atanh, 1) -> {float,[]}; -math_mod_return_type(erf, 1) -> {float,[]}; -math_mod_return_type(erfc, 1) -> {float,[]}; -math_mod_return_type(exp, 1) -> {float,[]}; -math_mod_return_type(log, 1) -> {float,[]}; -math_mod_return_type(log2, 1) -> {float,[]}; -math_mod_return_type(log10, 1) -> {float,[]}; -math_mod_return_type(sqrt, 1) -> {float,[]}; -math_mod_return_type(atan2, 2) -> {float,[]}; -math_mod_return_type(pow, 2) -> {float,[]}; -math_mod_return_type(ceil, 1) -> {float,[]}; -math_mod_return_type(floor, 1) -> {float,[]}; -math_mod_return_type(fmod, 2) -> {float,[]}; -math_mod_return_type(pi, 0) -> {float,[]}; -math_mod_return_type(F, A) when is_atom(F), is_integer(A), A >= 0 -> term. - -lists_mod_return_type(all, 2, _Vst) -> - bool; -lists_mod_return_type(any, 2, _Vst) -> - bool; -lists_mod_return_type(keymember, 3, _Vst) -> - bool; -lists_mod_return_type(member, 2, _Vst) -> - bool; -lists_mod_return_type(prefix, 2, _Vst) -> - bool; -lists_mod_return_type(suffix, 2, _Vst) -> - bool; -lists_mod_return_type(dropwhile, 2, _Vst) -> - list; -lists_mod_return_type(duplicate, 2, _Vst) -> - list; -lists_mod_return_type(filter, 2, _Vst) -> - list; -lists_mod_return_type(flatten, 1, _Vst) -> - list; -lists_mod_return_type(map, 2, Vst) -> - same_length_type({x,1}, Vst); -lists_mod_return_type(MF, 3, Vst) when MF =:= mapfoldl; MF =:= mapfoldr -> - ListType = same_length_type({x,2}, Vst), - {tuple,2,#{ {integer,1} => ListType} }; -lists_mod_return_type(partition, 2, _Vst) -> - two_tuple(list, list); -lists_mod_return_type(reverse, 1, Vst) -> - same_length_type({x,0}, Vst); -lists_mod_return_type(seq, 2, _Vst) -> - list; -lists_mod_return_type(sort, 1, Vst) -> - same_length_type({x,0}, Vst); -lists_mod_return_type(sort, 2, Vst) -> - same_length_type({x,1}, Vst); -lists_mod_return_type(splitwith, 2, _Vst) -> - two_tuple(list, list); -lists_mod_return_type(takewhile, 2, _Vst) -> - list; -lists_mod_return_type(unzip, 1, Vst) -> - ListType = same_length_type({x,0}, Vst), - two_tuple(ListType, ListType); -lists_mod_return_type(usort, 1, Vst) -> - same_length_type({x,0}, Vst); -lists_mod_return_type(zip, 2, _Vst) -> - list; -lists_mod_return_type(zipwith, 3, _Vst) -> - list; -lists_mod_return_type(_, _, _) -> - term. - -two_tuple(Type1, Type2) -> - {tuple,2,#{ {integer,1} => Type1, - {integer,2} => Type2 }}. - -same_length_type(Reg, Vst) -> - case get_term_type(Reg, Vst) of - {literal,[_|_]} -> cons; - cons -> cons; - nil -> nil; - _ -> list - end. +bif_types(Op, Ss, Vst) -> + Args = [normalize(get_term_type(Arg, Vst)) || Arg <- Ss], + beam_call_types:types(erlang, Op, Args). + +call_types({extfunc,M,F,A}, A, Vst) -> + Args = get_call_args(A, Vst), + beam_call_types:types(M, F, Args); +call_types(_, A, Vst) -> + {any, get_call_args(A, Vst), false}. + +will_bif_succeed(fadd, [_,_], _Vst) -> + maybe; +will_bif_succeed(fdiv, [_,_], _Vst) -> + maybe; +will_bif_succeed(fmul, [_,_], _Vst) -> + maybe; +will_bif_succeed(fnegate, [_], _Vst) -> + maybe; +will_bif_succeed(fsub, [_,_], _Vst) -> + maybe; +will_bif_succeed(Op, Ss, Vst) -> + Args = [normalize(get_term_type(Arg, Vst)) || Arg <- Ss], + beam_call_types:will_succeed(erlang, Op, Args). + +will_call_succeed({extfunc,M,F,A}, Vst) -> + beam_call_types:will_succeed(M, F, get_call_args(A, Vst)); +will_call_succeed(_Call, _Vst) -> + maybe. + +get_call_args(Arity, Vst) -> + get_call_args_1(0, Arity, Vst). + +get_call_args_1(Arity, Arity, _) -> + []; +get_call_args_1(N, Arity, Vst) when N < Arity -> + ArgType = normalize(get_movable_term_type({x,N}, Vst)), + [ArgType | get_call_args_1(N + 1, Arity, Vst)]. check_limit({x,X}=Src) when is_integer(X) -> if diff --git a/lib/compiler/src/cerl_sets.erl b/lib/compiler/src/cerl_sets.erl index f489baf238..84e488fc55 100644 --- a/lib/compiler/src/cerl_sets.erl +++ b/lib/compiler/src/cerl_sets.erl @@ -153,14 +153,21 @@ intersection1(S1, []) -> S1. Set1 :: set(Element), Set2 :: set(Element). -is_disjoint(S1, S2) when map_size(S1) < map_size(S2) -> - fold(fun (_, false) -> false; - (E, true) -> not is_element(E, S2) - end, true, S1); +is_disjoint(S1, S2) when map_size(S1) > map_size(S2) -> + is_disjoint_1(S1, maps:iterator(S2)); is_disjoint(S1, S2) -> - fold(fun (_, false) -> false; - (E, true) -> not is_element(E, S1) - end, true, S2). + is_disjoint_1(S2, maps:iterator(S1)). + +is_disjoint_1(Set, Iter) -> + case maps:next(Iter) of + {K, _, NextIter} -> + case Set of + #{K := _} -> false; + #{} -> is_disjoint_1(Set, NextIter) + end; + none -> + true + end. %% subtract(Set1, Set2) -> Set. %% Return all and only the elements of Set1 which are not also in @@ -180,8 +187,21 @@ subtract(S1, S2) -> Set1 :: set(Element), Set2 :: set(Element). +is_subset(S1, S2) when map_size(S1) > map_size(S2) -> + false; is_subset(S1, S2) -> - fold(fun (E, Sub) -> Sub andalso is_element(E, S2) end, true, S1). + is_subset_1(S2, maps:iterator(S1)). + +is_subset_1(Set, Iter) -> + case maps:next(Iter) of + {K, _, NextIter} -> + case Set of + #{K := _} -> is_subset_1(Set, NextIter); + #{} -> false + end; + none -> + true + end. %% fold(Fun, Accumulator, Set) -> Accumulator. %% Fold function Fun over all elements in Set and return Accumulator. @@ -193,8 +213,16 @@ is_subset(S1, S2) -> AccIn :: Acc, AccOut :: Acc. -fold(F, Init, D) -> - lists:foldl(fun(E,Acc) -> F(E,Acc) end,Init,maps:keys(D)). +fold(Fun, Init, Set) -> + fold_1(Fun, Init, maps:iterator(Set)). + +fold_1(Fun, Acc, Iter) -> + case maps:next(Iter) of + {K, _, NextIter} -> + fold_1(Fun, Fun(K,Acc), NextIter); + none -> + Acc + end. %% filter(Fun, Set) -> Set. %% Filter Set with Fun. @@ -203,5 +231,18 @@ fold(F, Init, D) -> Set1 :: set(Element), Set2 :: set(Element). -filter(F, D) -> - maps:filter(fun(K,_) -> F(K) end, D). +filter(Fun, Set) -> + maps:from_list(filter_1(Fun, maps:iterator(Set))). + +filter_1(Fun, Iter) -> + case maps:next(Iter) of + {K, _, NextIter} -> + case Fun(K) of + true -> + [{K,ok} | filter_1(Fun, NextIter)]; + false -> + filter_1(Fun, NextIter) + end; + none -> + [] + end. diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl index 0325c714d0..21d67f5649 100644 --- a/lib/compiler/src/compile.erl +++ b/lib/compiler/src/compile.erl @@ -265,7 +265,10 @@ expand_opt(r19, Os) -> expand_opt(r20, Os) -> expand_opt_before_21(Os); expand_opt(r21, Os) -> - [no_put_tuple2 | expand_opt(no_bsm3, Os)]; + [no_shared_fun_wrappers, + no_swap, no_put_tuple2 | expand_opt(no_bsm3, Os)]; +expand_opt(r22, Os) -> + [no_shared_fun_wrappers, no_swap | Os]; expand_opt({debug_info_key,_}=O, Os) -> [encrypt_debug_info,O|Os]; expand_opt(no_type_opt=O, Os) -> @@ -278,7 +281,8 @@ expand_opt(no_type_opt=O, Os) -> expand_opt(O, Os) -> [O|Os]. expand_opt_before_21(Os) -> - [no_put_tuple2, no_get_hd_tl, no_ssa_opt_record, + [no_shared_fun_wrappers, no_swap, + no_put_tuple2, no_get_hd_tl, no_ssa_opt_record, no_utf8_atoms | expand_opt(no_bsm3, Os)]. %% format_error(ErrorDescriptor) -> string() @@ -863,8 +867,6 @@ asm_passes() -> {unless,no_postopt, [{pass,beam_block}, {iff,dblk,{listing,"block"}}, - {unless,no_except,{pass,beam_except}}, - {iff,dexcept,{listing,"except"}}, {unless,no_jopt,{pass,beam_jump}}, {iff,djmp,{listing,"jump"}}, {unless,no_peep_opt,{pass,beam_peep}}, @@ -2096,9 +2098,9 @@ pre_load() -> L = [beam_a, beam_asm, beam_block, + beam_call_types, beam_clean, beam_dict, - beam_except, beam_flatten, beam_jump, beam_kernel_to_ssa, @@ -2115,6 +2117,7 @@ pre_load() -> beam_ssa_share, beam_ssa_type, beam_trim, + beam_types, beam_utils, beam_validator, beam_z, diff --git a/lib/compiler/src/compiler.app.src b/lib/compiler/src/compiler.app.src index a086a3a8d3..092757ae65 100644 --- a/lib/compiler/src/compiler.app.src +++ b/lib/compiler/src/compiler.app.src @@ -24,10 +24,10 @@ beam_a, beam_asm, beam_block, + beam_call_types, beam_clean, beam_dict, beam_disasm, - beam_except, beam_flatten, beam_jump, beam_kernel_to_ssa, @@ -47,6 +47,7 @@ beam_ssa_share, beam_ssa_type, beam_trim, + beam_types, beam_utils, beam_validator, beam_z, diff --git a/lib/compiler/src/erl_bifs.erl b/lib/compiler/src/erl_bifs.erl index 94a5dfe012..1b60995de9 100644 --- a/lib/compiler/src/erl_bifs.erl +++ b/lib/compiler/src/erl_bifs.erl @@ -80,10 +80,12 @@ is_pure(erlang, 'or', 2) -> true; is_pure(erlang, 'rem', 2) -> true; is_pure(erlang, 'xor', 2) -> true; is_pure(erlang, abs, 1) -> true; +is_pure(erlang, atom_to_binary, 1) -> true; is_pure(erlang, atom_to_binary, 2) -> true; is_pure(erlang, atom_to_list, 1) -> true; is_pure(erlang, binary_part, 2) -> true; is_pure(erlang, binary_part, 3) -> true; +is_pure(erlang, binary_to_atom, 1) -> true; is_pure(erlang, binary_to_atom, 2) -> true; is_pure(erlang, binary_to_float, 1) -> true; is_pure(erlang, binary_to_integer, 1) -> true; diff --git a/lib/compiler/src/genop.tab b/lib/compiler/src/genop.tab index 86590fad87..0a38d17857 100755 --- a/lib/compiler/src/genop.tab +++ b/lib/compiler/src/genop.tab @@ -596,3 +596,9 @@ BEAM_FORMAT_NUMBER=0 ## @spec bs_set_positon Ctx Pos ## @doc Sets the current position of Ctx to Pos 168: bs_set_position/2 + +# OTP 23 + +## @spec swap Register1 Register2 +## @doc Swaps the contents of two registers. +169: swap/2 diff --git a/lib/compiler/src/sys_core_fold.erl b/lib/compiler/src/sys_core_fold.erl index 4939a94a92..63c67639d4 100644 --- a/lib/compiler/src/sys_core_fold.erl +++ b/lib/compiler/src/sys_core_fold.erl @@ -99,10 +99,6 @@ t=#{} :: map(), %Types in_guard=false}). %In guard or not. --type type_info() :: cerl:cerl() | 'bool' | 'integer' | {'fun', pos_integer()}. --type yes_no_maybe() :: 'yes' | 'no' | 'maybe'. --type sub() :: #sub{}. - -spec module(cerl:c_module(), [compile:option()]) -> {'ok', cerl:c_module(), [_]}. @@ -315,10 +311,10 @@ expr(#c_seq{arg=Arg0,body=B0}=Seq0, Ctxt, Sub) -> false -> %% Arg cannot be "values" here - only a single value %% make sense here. - case {Ctxt,is_safe_simple(Arg, Sub)} of + case {Ctxt,is_safe_simple(Arg)} of {effect,true} -> B1; {effect,false} -> - case is_safe_simple(B1, Sub) of + case is_safe_simple(B1) of true -> Arg; false -> Seq0#c_seq{arg=Arg,body=B1} end; @@ -442,7 +438,7 @@ expr(#c_catch{anno=Anno,body=B}, effect, Sub) -> expr(#c_catch{body=B0}=Catch, _, Sub) -> %% We can remove catch if the value is simple B1 = body(B0, value, Sub), - case is_safe_simple(B1, Sub) of + case is_safe_simple(B1) of true -> B1; false -> Catch#c_catch{body=B1} end; @@ -458,7 +454,7 @@ expr(#c_try{arg=E0,vars=[#c_var{name=X}],body=#c_var{name=X}, %% We can remove try/catch if the expression is an %% expression that cannot fail. - case is_safe_bool_expr(E2, Sub) orelse is_safe_simple(E2, Sub) of + case is_safe_bool_expr(E2) orelse is_safe_simple(E2) of true -> E2; false -> Try#c_try{arg=E2} end; @@ -472,7 +468,7 @@ expr(#c_try{anno=A,arg=E0,vars=Vs0,body=B0,evars=Evs0,handler=H0}=Try, _, Sub0) E1 = body(E0, value, Sub0), {Vs1,Sub1} = var_list(Vs0, Sub0), B1 = body(B0, value, Sub1), - case is_safe_simple(E1, Sub0) of + case is_safe_simple(E1) of true -> expr(#c_let{anno=A,vars=Vs1,arg=E1,body=B1}, value, Sub0); false -> @@ -602,20 +598,20 @@ is_literal_fun(_) -> false. %% Currently, we don't attempt to check binaries because they %% are difficult to check. -is_safe_simple(#c_var{}=Var, _) -> +is_safe_simple(#c_var{}=Var) -> not cerl:is_c_fname(Var); -is_safe_simple(#c_cons{hd=H,tl=T}, Sub) -> - is_safe_simple(H, Sub) andalso is_safe_simple(T, Sub); -is_safe_simple(#c_tuple{es=Es}, Sub) -> is_safe_simple_list(Es, Sub); -is_safe_simple(#c_literal{}, _) -> true; +is_safe_simple(#c_cons{hd=H,tl=T}) -> + is_safe_simple(H) andalso is_safe_simple(T); +is_safe_simple(#c_tuple{es=Es}) -> is_safe_simple_list(Es); +is_safe_simple(#c_literal{}) -> true; is_safe_simple(#c_call{module=#c_literal{val=erlang}, name=#c_literal{val=Name}, - args=Args}, Sub) when is_atom(Name) -> + args=Args}) when is_atom(Name) -> NumArgs = length(Args), case erl_internal:bool_op(Name, NumArgs) of true -> %% Boolean operators are safe if the arguments are boolean. - all(fun(C) -> is_boolean_type(C, Sub) =:= yes end, Args); + all(fun is_bool_expr/1, Args); false -> %% We need a rather complicated test to ensure that %% we only allow safe calls that are allowed in a guard. @@ -624,9 +620,9 @@ is_safe_simple(#c_call{module=#c_literal{val=erlang}, (erl_internal:comp_op(Name, NumArgs) orelse erl_internal:new_type_test(Name, NumArgs)) end; -is_safe_simple(_, _) -> false. +is_safe_simple(_) -> false. -is_safe_simple_list(Es, Sub) -> all(fun(E) -> is_safe_simple(E, Sub) end, Es). +is_safe_simple_list(Es) -> all(fun(E) -> is_safe_simple(E) end, Es). %% will_fail(Expr) -> true|false. %% Determine whether the expression will fail with an exception. @@ -853,7 +849,7 @@ useless_call(_, _) -> no. %% Anything that will not have any effect will be thrown away. make_effect_seq([H|T], Sub) -> - case is_safe_simple(H, Sub) of + case is_safe_simple(H) of true -> make_effect_seq(T, Sub); false -> #c_seq{arg=H,body=make_effect_seq(T, Sub)} end; @@ -959,138 +955,14 @@ fold_lit_args(Call, Module, Name, Args0) -> %% Attempt to evaluate some pure BIF calls with one or more %% non-literals arguments. %% -fold_non_lit_args(Call, erlang, is_boolean, [Arg], Sub) -> - eval_is_boolean(Call, Arg, Sub); fold_non_lit_args(Call, erlang, length, [Arg], _) -> eval_length(Call, Arg); fold_non_lit_args(Call, erlang, '++', [Arg1,Arg2], _) -> eval_append(Call, Arg1, Arg2); fold_non_lit_args(Call, lists, append, [Arg1,Arg2], _) -> eval_append(Call, Arg1, Arg2); -fold_non_lit_args(Call, erlang, is_function, [Arg1], Sub) -> - eval_is_function_1(Call, Arg1, Sub); -fold_non_lit_args(Call, erlang, is_function, [Arg1,Arg2], Sub) -> - eval_is_function_2(Call, Arg1, Arg2, Sub); -fold_non_lit_args(Call, erlang, N, Args, Sub) -> - NumArgs = length(Args), - case erl_internal:comp_op(N, NumArgs) of - true -> - eval_rel_op(Call, N, Args, Sub); - false -> - case erl_internal:bool_op(N, NumArgs) of - true -> - eval_bool_op(Call, N, Args, Sub); - false -> - Call - end - end; fold_non_lit_args(Call, _, _, _, _) -> Call. -eval_is_function_1(Call, Arg1, Sub) -> - case get_type(Arg1, Sub) of - none -> Call; - {'fun',_} -> #c_literal{anno=cerl:get_ann(Call),val=true}; - _ -> #c_literal{anno=cerl:get_ann(Call),val=false} - end. - -eval_is_function_2(Call, Arg1, #c_literal{val=Arity}, Sub) - when is_integer(Arity), Arity > 0 -> - case get_type(Arg1, Sub) of - none -> Call; - {'fun',Arity} -> #c_literal{anno=cerl:get_ann(Call),val=true}; - _ -> #c_literal{anno=cerl:get_ann(Call),val=false} - end; -eval_is_function_2(Call, _Arg1, _Arg2, _Sub) -> Call. - -%% Evaluate a relational operation using type information. -eval_rel_op(Call, Op, [#c_var{name=V},#c_var{name=V}], _) -> - Bool = erlang:Op(same, same), - #c_literal{anno=cerl:get_ann(Call),val=Bool}; -eval_rel_op(Call, '=:=', [Term,#c_literal{val=true}], Sub) -> - %% BoolVar =:= true ==> BoolVar - case is_boolean_type(Term, Sub) of - yes -> Term; - maybe -> Call; - no -> #c_literal{val=false} - end; -eval_rel_op(Call, '==', Ops, Sub) -> - case is_exact_eq_ok(Ops, Sub) of - true -> - Name = #c_literal{anno=cerl:get_ann(Call),val='=:='}, - Call#c_call{name=Name}; - false -> - Call - end; -eval_rel_op(Call, '/=', Ops, Sub) -> - case is_exact_eq_ok(Ops, Sub) of - true -> - Name = #c_literal{anno=cerl:get_ann(Call),val='=/='}, - Call#c_call{name=Name}; - false -> - Call - end; -eval_rel_op(Call, _, _, _) -> Call. - -is_exact_eq_ok([A,B]=L, Sub) -> - case is_int_type(A, Sub) =:= yes andalso is_int_type(B, Sub) =:= yes of - true -> true; - false -> is_exact_eq_ok_1(L) - end. - -is_exact_eq_ok_1([#c_literal{val=Lit}|_]) -> - is_non_numeric(Lit); -is_exact_eq_ok_1([_|T]) -> - is_exact_eq_ok_1(T); -is_exact_eq_ok_1([]) -> false. - -is_non_numeric([H|T]) -> - is_non_numeric(H) andalso is_non_numeric(T); -is_non_numeric(Tuple) when is_tuple(Tuple) -> - is_non_numeric_tuple(Tuple, tuple_size(Tuple)); -is_non_numeric(Map) when is_map(Map) -> - %% Note that 17.x and 18.x compare keys in different ways. - %% Be very conservative -- require that both keys and values - %% are non-numeric. - is_non_numeric(maps:to_list(Map)); -is_non_numeric(Num) when is_number(Num) -> - false; -is_non_numeric(_) -> true. - -is_non_numeric_tuple(Tuple, El) when El >= 1 -> - is_non_numeric(element(El, Tuple)) andalso - is_non_numeric_tuple(Tuple, El-1); -is_non_numeric_tuple(_Tuple, 0) -> true. - -%% Evaluate a bool op using type information. We KNOW that -%% there must be at least one non-literal argument (i.e. -%% there is no need to handle the case that all argments -%% are literal). - -eval_bool_op(Call, 'and', [#c_literal{val=true},Term], Sub) -> - eval_bool_op_1(Call, Term, Term, Sub); -eval_bool_op(Call, 'and', [Term,#c_literal{val=true}], Sub) -> - eval_bool_op_1(Call, Term, Term, Sub); -eval_bool_op(Call, 'and', [#c_literal{val=false}=Res,Term], Sub) -> - eval_bool_op_1(Call, Res, Term, Sub); -eval_bool_op(Call, 'and', [Term,#c_literal{val=false}=Res], Sub) -> - eval_bool_op_1(Call, Res, Term, Sub); -eval_bool_op(Call, _, _, _) -> Call. - -eval_bool_op_1(Call, Res, Term, Sub) -> - case is_boolean_type(Term, Sub) of - yes -> Res; - no -> eval_failure(Call, badarg); - maybe -> Call - end. - -%% Evaluate is_boolean/1 using type information. -eval_is_boolean(Call, Term, Sub) -> - case is_boolean_type(Term, Sub) of - no -> #c_literal{val=false}; - yes -> #c_literal{val=true}; - maybe -> Call - end. - %% eval_length(Call, List) -> Val. %% Evaluates the length for the prefix of List which has a known %% shape. @@ -1804,7 +1676,7 @@ opt_bool_case_guard(#c_case{arg=#c_literal{}}=Case) -> %% Case; opt_bool_case_guard(#c_case{arg=Arg,clauses=Cs0}=Case) -> - case is_safe_bool_expr(Arg, sub_new()) of + case is_safe_bool_expr(Arg) of false -> Case; true -> @@ -1945,7 +1817,7 @@ case_opt_arg(E0, Sub, Cs, LitExpr) -> {error,Cs}; false -> %% If possible, expand this variable to a previously - %% matched term. + %% constructed tuple E = case_expand_var(E0, Sub), case_opt_arg_1(E, Cs, LitExpr) end @@ -2004,13 +1876,8 @@ case_opt_compiler_generated(Core) -> case_expand_var(E, #sub{t=Tdb}) -> Key = cerl:var_name(E), case Tdb of - #{Key:=T} -> - case cerl:is_c_tuple(T) of - false -> E; - true -> T - end; - _ -> - E + #{Key:=T} -> T; + _ -> E end. %% case_opt_nomatch(E, Clauses, LitExpr) -> Clauses' @@ -2302,43 +2169,30 @@ is_simple_case_arg(_) -> false. %% Check whether the Core expression is guaranteed to return %% a boolean IF IT RETURNS AT ALL. %% -is_bool_expr(Core) -> - is_bool_expr(Core, sub_new()). -%% is_bool_expr(Core, Sub) -> true|false -%% Check whether the Core expression is guaranteed to return -%% a boolean IF IT RETURNS AT ALL. Uses type information -%% to be able to identify more expressions as booleans. -%% is_bool_expr(#c_call{module=#c_literal{val=erlang}, - name=#c_literal{val=Name},args=Args}=Call, _) -> + name=#c_literal{val=Name},args=Args}=Call) -> NumArgs = length(Args), erl_internal:comp_op(Name, NumArgs) orelse erl_internal:new_type_test(Name, NumArgs) orelse erl_internal:bool_op(Name, NumArgs) orelse will_fail(Call); is_bool_expr(#c_try{arg=E,vars=[#c_var{name=X}],body=#c_var{name=X}, - handler=#c_literal{val=false}}, Sub) -> - is_bool_expr(E, Sub); -is_bool_expr(#c_case{clauses=Cs}, Sub) -> - is_bool_expr_list(Cs, Sub); -is_bool_expr(#c_clause{body=B}, Sub) -> - is_bool_expr(B, Sub); -is_bool_expr(#c_let{vars=[V],arg=Arg,body=B}, Sub0) -> - Sub = case is_bool_expr(Arg, Sub0) of - true -> update_types(V, [bool], Sub0); - false -> Sub0 - end, - is_bool_expr(B, Sub); -is_bool_expr(#c_let{body=B}, Sub) -> - %% Binding of multiple variables. - is_bool_expr(B, Sub); -is_bool_expr(C, Sub) -> - is_boolean_type(C, Sub) =:= yes. - -is_bool_expr_list([C|Cs], Sub) -> - is_bool_expr(C, Sub) andalso is_bool_expr_list(Cs, Sub); -is_bool_expr_list([], _) -> true. + handler=#c_literal{val=false}}) -> + is_bool_expr(E); +is_bool_expr(#c_case{clauses=Cs}) -> + is_bool_expr_list(Cs); +is_bool_expr(#c_clause{body=B}) -> + is_bool_expr(B); +is_bool_expr(#c_let{body=B}) -> + is_bool_expr(B); +is_bool_expr(#c_literal{val=Val}) -> + is_boolean(Val); +is_bool_expr(_) -> false. + +is_bool_expr_list([C|Cs]) -> + is_bool_expr(C) andalso is_bool_expr_list(Cs); +is_bool_expr_list([]) -> true. %% is_safe_bool_expr(Core) -> true|false %% Check whether the Core expression ALWAYS returns a boolean @@ -2346,17 +2200,17 @@ is_bool_expr_list([], _) -> true. %% is suitable for a guard (no calls to non-guard BIFs, local %% functions, or is_record/2). %% -is_safe_bool_expr(Core, Sub) -> - is_safe_bool_expr_1(Core, Sub, cerl_sets:new()). +is_safe_bool_expr(Core) -> + is_safe_bool_expr_1(Core, cerl_sets:new()). is_safe_bool_expr_1(#c_call{module=#c_literal{val=erlang}, name=#c_literal{val=is_record}, args=[A,#c_literal{val=Tag},#c_literal{val=Size}]}, - Sub, _BoolVars) when is_atom(Tag), is_integer(Size) -> - is_safe_simple(A, Sub); + _BoolVars) when is_atom(Tag), is_integer(Size) -> + is_safe_simple(A); is_safe_bool_expr_1(#c_call{module=#c_literal{val=erlang}, name=#c_literal{val=is_record}}, - _Sub, _BoolVars) -> + _BoolVars) -> %% The is_record/2 BIF is NOT allowed in guards. %% The is_record/3 BIF where its second argument is not an atom or its third %% is not an integer is NOT allowed in guards. @@ -2368,49 +2222,49 @@ is_safe_bool_expr_1(#c_call{module=#c_literal{val=erlang}, is_safe_bool_expr_1(#c_call{module=#c_literal{val=erlang}, name=#c_literal{val=is_function}, args=[A,#c_literal{val=Arity}]}, - Sub, _BoolVars) when is_integer(Arity), Arity >= 0 -> - is_safe_simple(A, Sub); + _BoolVars) when is_integer(Arity), Arity >= 0 -> + is_safe_simple(A); is_safe_bool_expr_1(#c_call{module=#c_literal{val=erlang}, name=#c_literal{val=is_function}}, - _Sub, _BoolVars) -> + _BoolVars) -> false; is_safe_bool_expr_1(#c_call{module=#c_literal{val=erlang}, name=#c_literal{val=Name},args=Args}, - Sub, BoolVars) -> + BoolVars) -> NumArgs = length(Args), case (erl_internal:comp_op(Name, NumArgs) orelse erl_internal:new_type_test(Name, NumArgs)) andalso - is_safe_simple_list(Args, Sub) of + is_safe_simple_list(Args) of true -> true; false -> %% Boolean operators are safe if all arguments are boolean. erl_internal:bool_op(Name, NumArgs) andalso - is_safe_bool_expr_list(Args, Sub, BoolVars) + is_safe_bool_expr_list(Args, BoolVars) end; -is_safe_bool_expr_1(#c_let{vars=Vars,arg=Arg,body=B}, Sub, BoolVars) -> - case is_safe_simple(Arg, Sub) of +is_safe_bool_expr_1(#c_let{vars=Vars,arg=Arg,body=B}, BoolVars) -> + case is_safe_simple(Arg) of true -> - case {is_safe_bool_expr_1(Arg, Sub, BoolVars),Vars} of + case {is_safe_bool_expr_1(Arg, BoolVars),Vars} of {true,[#c_var{name=V}]} -> - is_safe_bool_expr_1(B, Sub, cerl_sets:add_element(V, BoolVars)); + is_safe_bool_expr_1(B, cerl_sets:add_element(V, BoolVars)); {false,_} -> - is_safe_bool_expr_1(B, Sub, BoolVars) + is_safe_bool_expr_1(B, BoolVars) end; false -> false end; -is_safe_bool_expr_1(#c_literal{val=Val}, _Sub, _) -> +is_safe_bool_expr_1(#c_literal{val=Val}, _BoolVars) -> is_boolean(Val); -is_safe_bool_expr_1(#c_var{name=V}, _Sub, BoolVars) -> +is_safe_bool_expr_1(#c_var{name=V}, BoolVars) -> cerl_sets:is_element(V, BoolVars); -is_safe_bool_expr_1(_, _, _) -> false. +is_safe_bool_expr_1(_, _) -> false. -is_safe_bool_expr_list([C|Cs], Sub, BoolVars) -> - case is_safe_bool_expr_1(C, Sub, BoolVars) of - true -> is_safe_bool_expr_list(Cs, Sub, BoolVars); +is_safe_bool_expr_list([C|Cs], BoolVars) -> + case is_safe_bool_expr_1(C, BoolVars) of + true -> is_safe_bool_expr_list(Cs, BoolVars); false -> false end; -is_safe_bool_expr_list([], _, _) -> true. +is_safe_bool_expr_list([], _) -> true. %% simplify_let(Let, Sub) -> Expr | impossible %% If the argument part of an let contains a complex expression, such @@ -2785,7 +2639,7 @@ opt_simple_let_2(Let0, Vs0, Arg0, Body, PrevBody, Sub) -> %% with exported variables, but the return value is %% ignored). We can remove the first variable and the %% the first value returned from the 'let' argument. - Arg2 = remove_first_value(Arg1, Sub), + Arg2 = remove_first_value(Arg1), Let1 = Let0#c_let{vars=Vars,arg=Arg2,body=Body}, post_opt_let(Let1, Sub); true -> @@ -2805,36 +2659,36 @@ post_opt_let(Let0, Sub) -> opt_build_stacktrace(Let1). -%% remove_first_value(Core0, Sub) -> Core. +%% remove_first_value(Core0) -> Core. %% Core0 is an expression that returns at least two values. %% Remove the first value returned from Core0. -remove_first_value(#c_values{es=[V|Vs]}, Sub) -> +remove_first_value(#c_values{es=[V|Vs]}) -> Values = core_lib:make_values(Vs), - case is_safe_simple(V, Sub) of + case is_safe_simple(V) of false -> #c_seq{arg=V,body=Values}; true -> Values end; -remove_first_value(#c_case{clauses=Cs0}=Core, Sub) -> - Cs = remove_first_value_cs(Cs0, Sub), +remove_first_value(#c_case{clauses=Cs0}=Core) -> + Cs = remove_first_value_cs(Cs0), Core#c_case{clauses=Cs}; -remove_first_value(#c_receive{clauses=Cs0,action=Act0}=Core, Sub) -> - Cs = remove_first_value_cs(Cs0, Sub), - Act = remove_first_value(Act0, Sub), +remove_first_value(#c_receive{clauses=Cs0,action=Act0}=Core) -> + Cs = remove_first_value_cs(Cs0), + Act = remove_first_value(Act0), Core#c_receive{clauses=Cs,action=Act}; -remove_first_value(#c_let{body=B}=Core, Sub) -> - Core#c_let{body=remove_first_value(B, Sub)}; -remove_first_value(#c_seq{body=B}=Core, Sub) -> - Core#c_seq{body=remove_first_value(B, Sub)}; -remove_first_value(#c_primop{}=Core, _Sub) -> +remove_first_value(#c_let{body=B}=Core) -> + Core#c_let{body=remove_first_value(B)}; +remove_first_value(#c_seq{body=B}=Core) -> + Core#c_seq{body=remove_first_value(B)}; +remove_first_value(#c_primop{}=Core) -> Core; -remove_first_value(#c_call{}=Core, _Sub) -> +remove_first_value(#c_call{}=Core) -> Core. -remove_first_value_cs(Cs, Sub) -> - [C#c_clause{body=remove_first_value(B, Sub)} || +remove_first_value_cs(Cs) -> + [C#c_clause{body=remove_first_value(B)} || #c_clause{body=B}=C <- Cs]. %% maybe_suppress_warnings(Arg, #c_var{}, PreviousBody) -> Arg' @@ -2962,54 +2816,6 @@ move_case_into_arg(Expr, _) -> Expr. %%% -%%% Retrieving information about types. -%%% - --spec get_type(cerl:cerl(), #sub{}) -> type_info() | 'none'. - -get_type(#c_var{name=V}, #sub{t=Tdb}) -> - case Tdb of - #{V:=Type} -> Type; - _ -> none - end; -get_type(C, _) -> - case cerl:type(C) of - binary -> C; - map -> C; - _ -> - case cerl:is_data(C) of - true -> C; - false -> none - end - end. - --spec is_boolean_type(cerl:cerl(), sub()) -> yes_no_maybe(). - -is_boolean_type(Var, Sub) -> - case get_type(Var, Sub) of - none -> - maybe; - bool -> - yes; - C -> - B = cerl:is_c_atom(C) andalso - is_boolean(cerl:atom_val(C)), - yes_no(B) - end. - --spec is_int_type(cerl:cerl(), sub()) -> yes_no_maybe(). - -is_int_type(Var, Sub) -> - case get_type(Var, Sub) of - none -> maybe; - integer -> yes; - C -> yes_no(cerl:is_c_int(C)) - end. - -yes_no(true) -> yes; -yes_no(false) -> no. - -%%% %%% Update type information. %%% @@ -3020,70 +2826,14 @@ update_let_types(_Vs, _Arg, Sub) -> %% that returns multiple values. Sub. -update_let_types_1([#c_var{}=V|Vs], [A|As], Sub0) -> - Sub = update_types_from_expr(V, A, Sub0), +update_let_types_1([#c_var{name=V}|Vs], [A|As], Sub0) -> + Sub = update_types(V, A, Sub0), update_let_types_1(Vs, As, Sub); update_let_types_1([], [], Sub) -> Sub. -update_types_from_expr(V, Expr, Sub) -> - Type = extract_type(Expr, Sub), - update_types(V, [Type], Sub). - -extract_type(#c_call{module=#c_literal{val=erlang}, - name=#c_literal{val=Name}, - args=Args}=Call, Sub) -> - case returns_integer(Name, Args) of - true -> integer; - false -> extract_type_1(Call, Sub) - end; -extract_type(Expr, Sub) -> - extract_type_1(Expr, Sub). - -extract_type_1(Expr, Sub) -> - case is_bool_expr(Expr, Sub) of - false -> Expr; - true -> bool - end. - -returns_integer('band', [_,_]) -> true; -returns_integer('bnot', [_]) -> true; -returns_integer('bor', [_,_]) -> true; -returns_integer('bxor', [_,_]) -> true; -returns_integer(bit_size, [_]) -> true; -returns_integer('bsl', [_,_]) -> true; -returns_integer('bsr', [_,_]) -> true; -returns_integer(byte_size, [_]) -> true; -returns_integer(ceil, [_]) -> true; -returns_integer('div', [_,_]) -> true; -returns_integer(floor, [_]) -> true; -returns_integer(length, [_]) -> true; -returns_integer('rem', [_,_]) -> true; -returns_integer('round', [_]) -> true; -returns_integer(size, [_]) -> true; -returns_integer(tuple_size, [_]) -> true; -returns_integer(trunc, [_]) -> true; -returns_integer(_, _) -> false. - -%% update_types(Expr, Pattern, Sub) -> Sub' -%% Update the type database. - --spec update_types(cerl:c_var(), [type_info()], sub()) -> sub(). - -update_types(#c_var{name=V}, Pat, #sub{t=Tdb0}=Sub) -> - Tdb = update_types_1(V, Pat, Tdb0), - Sub#sub{t=Tdb}. - -update_types_1(V, [#c_tuple{}=P], Types) -> - Types#{V=>P}; -update_types_1(V, [#c_literal{val=Bool}], Types) when is_boolean(Bool) -> - Types#{V=>bool}; -update_types_1(V, [#c_fun{vars=Vars}], Types) -> - Types#{V=>{'fun',length(Vars)}}; -update_types_1(V, [#c_var{name={_,Arity}}], Types) -> - Types#{V=>{'fun',Arity}}; -update_types_1(V, [Type], Types) when is_atom(Type) -> - Types#{V=>Type}; -update_types_1(_, _, Types) -> Types. +update_types(V, #c_tuple{}=P, #sub{t=Tdb}=Sub) -> + Sub#sub{t=Tdb#{V=>P}}; +update_types(_, _, Sub) -> Sub. %% kill_types(V, Tdb) -> Tdb' %% Kill any entries that references the variable, @@ -3099,10 +2849,6 @@ kill_types2(V, [{_,#c_tuple{}=Tuple}=Entry|Tdb]) -> false -> [Entry|kill_types2(V, Tdb)]; true -> kill_types2(V, Tdb) end; -kill_types2(V, [{_, {'fun',_}}=Entry|Tdb]) -> - [Entry|kill_types2(V, Tdb)]; -kill_types2(V, [{_,Atom}=Entry|Tdb]) when is_atom(Atom) -> - [Entry|kill_types2(V, Tdb)]; kill_types2(_, []) -> []. %% copy_type(DestVar, SrcVar, Tdb) -> Tdb' diff --git a/lib/compiler/src/v3_kernel.erl b/lib/compiler/src/v3_kernel.erl index e2b8787224..49fb66126f 100644 --- a/lib/compiler/src/v3_kernel.erl +++ b/lib/compiler/src/v3_kernel.erl @@ -81,13 +81,17 @@ -export([module/2,format_error/1]). --import(lists, [map/2,foldl/3,foldr/3,mapfoldl/3,splitwith/2,member/2, - keyfind/3,partition/2,droplast/1,last/1,sort/1,reverse/1]). +-import(lists, [droplast/1,flatten/1,foldl/3,foldr/3, + map/2,mapfoldl/3,member/2, + keyfind/3,keyreplace/4, + last/1,partition/2,reverse/1, + splitwith/2,sort/1]). -import(ordsets, [add_element/2,del_element/2,union/2,union/1,subtract/2]). -import(cerl, [c_tuple/1]). -include("core_parse.hrl"). -include("v3_kernel.hrl"). +-define(EXPAND_MAX_SIZE_SEGMENT, 1024). %% These are not defined in v3_kernel.hrl. get_kanno(Kthing) -> element(2, Kthing). @@ -120,15 +124,19 @@ copy_anno(Kdst, Ksrc) -> funs=[], %Fun functions free=#{}, %Free variables ws=[] :: [warning()], %Warnings. - guard_refc=0}). %> 0 means in guard + guard_refc=0, %> 0 means in guard + no_shared_fun_wrappers=false :: boolean() + }). -spec module(cerl:c_module(), [compile:option()]) -> {'ok', #k_mdef{}, [warning()]}. -module(#c_module{anno=A,name=M,exports=Es,attrs=As,defs=Fs}, _Options) -> +module(#c_module{anno=A,name=M,exports=Es,attrs=As,defs=Fs}, Options) -> Kas = attributes(As), Kes = map(fun (#c_var{name={_,_}=Fname}) -> Fname end, Es), - St0 = #kern{}, + NoSharedFunWrappers = proplists:get_bool(no_shared_fun_wrappers, + Options), + St0 = #kern{no_shared_fun_wrappers=NoSharedFunWrappers}, {Kfs,St} = mapfoldl(fun function/2, St0, Fs), {ok,#k_mdef{anno=A,name=M#c_literal.val,exports=Kes,attributes=Kas, body=Kfs ++ St#kern.funs},lists:sort(St#kern.ws)}. @@ -716,16 +724,27 @@ gexpr_test_add(Ke, St0) -> %% expr(Cexpr, Sub, State) -> {Kexpr,[PreKexpr],State}. %% Convert a Core expression, flattening it at the same time. -expr(#c_var{anno=A,name={_Name,Arity}}=Fname, Sub, St) -> - %% A local in an expression. - %% For now, these are wrapped into a fun by reverse - %% eta-conversion, but really, there should be exactly one - %% such "lambda function" for each escaping local name, - %% instead of one for each occurrence as done now. +expr(#c_var{anno=A0,name={Name,Arity}}=Fname, Sub, St) -> Vs = [#c_var{name=list_to_atom("V" ++ integer_to_list(V))} || - V <- integers(1, Arity)], - Fun = #c_fun{anno=A,vars=Vs,body=#c_apply{anno=A,op=Fname,args=Vs}}, - expr(Fun, Sub, St); + V <- integers(1, Arity)], + case St#kern.no_shared_fun_wrappers of + false -> + %% Generate a (possibly shared) wrapper function for calling + %% this function. + Wrapper0 = ["-fun.",atom_to_list(Name),"/",integer_to_list(Arity),"-"], + Wrapper = list_to_atom(flatten(Wrapper0)), + Id = {id,{0,0,Wrapper}}, + A = keyreplace(id, 1, A0, Id), + Fun = #c_fun{anno=A,vars=Vs,body=#c_apply{anno=A,op=Fname,args=Vs}}, + expr(Fun, Sub, St); + true -> + %% For backward compatibility with OTP 22 and earlier, + %% use the pre-generated name for the fun wrapper. + %% There will be one wrapper function for each occurrence + %% of `fun F/A`. + Fun = #c_fun{anno=A0,vars=Vs,body=#c_apply{anno=A0,op=Fname,args=Vs}}, + expr(Fun, Sub, St) + end; expr(#c_var{anno=A,name=V}, Sub, St) -> {#k_var{anno=A,name=get_vsub(V, Sub)},[],St}; expr(#c_literal{anno=A,val=V}, _Sub, St) -> @@ -1152,7 +1171,7 @@ validate_bin_element_size(#k_int{val=V}) when V >= 0 -> ok; validate_bin_element_size(#k_atom{val=all}) -> ok; validate_bin_element_size(#k_atom{val=undefined}) -> ok; validate_bin_element_size(_) -> throw(bad_element_size). - + %% atomic_list([Cexpr], Sub, State) -> {[Kexpr],[PreKexpr],State}. atomic_list(Ces, Sub, St) -> @@ -1278,14 +1297,63 @@ pattern_bin_1([#c_bitstr{anno=A,val=E0,size=S0,unit=U,type=T,flags=Fs}|Es0], _ -> Isub0 end, {Es,{Isub,Osub},St3} = pattern_bin_1(Es0, Isub1, Osub1, St2), - {#k_bin_seg{anno=A,size=S, - unit=U0, - type=cerl:concrete(T), - flags=Fs0, - seg=E,next=Es}, - {Isub,Osub},St3}; + {build_bin_seg(A, S, U0, cerl:concrete(T), Fs0, E, Es),{Isub,Osub},St3}; pattern_bin_1([], Isub, Osub, St) -> {#k_bin_end{},{Isub,Osub},St}. +%% build_bin_seg(Anno, Size, Unit, Type, Flags, Seg, Next) -> #k_bin_seg{}. +%% This function normalizes literal integers with size > 8 and literal +%% utf8 segments into integers with size = 8 (and potentially an integer +%% with size less than 8 at the end). This is so further optimizations +%% have a normalized view of literal integers, allowing us to generate +%% more literals and group more clauses. Those integers may be "squeezed" +%% later into the largest integer possible. +%% +build_bin_seg(A, #k_int{val=Bits} = Sz, U, integer=Type, [unsigned,big]=Flags, #k_literal{val=Int}=Seg, Next) -> + Size = Bits * U, + case integer_fits_and_is_expandable(Int, Size) of + true -> build_bin_seg_integer_recur(A, Size, Int, Next); + false -> #k_bin_seg{anno=A,size=Sz,unit=U,type=Type,flags=Flags,seg=Seg,next=Next} + end; +build_bin_seg(A, Sz, U, utf8=Type, [unsigned,big]=Flags, #k_literal{val=Utf8} = Seg, Next) -> + case utf8_fits(Utf8) of + {Int, Bits} -> build_bin_seg_integer_recur(A, Bits, Int, Next); + error -> #k_bin_seg{anno=A,size=Sz,unit=U,type=Type,flags=Flags,seg=Seg,next=Next} + end; +build_bin_seg(A, Sz, U, Type, Flags, Seg, Next) -> + #k_bin_seg{anno=A,size=Sz,unit=U,type=Type,flags=Flags,seg=Seg,next=Next}. + +build_bin_seg_integer_recur(A, Bits, Val, Next) when Bits > 8 -> + NextBits = Bits - 8, + NextVal = Val band ((1 bsl NextBits) - 1), + Last = build_bin_seg_integer_recur(A, NextBits, NextVal, Next), + build_bin_seg_integer(A, 8, Val bsr NextBits, Last); + +build_bin_seg_integer_recur(A, Bits, Val, Next) -> + build_bin_seg_integer(A, Bits, Val, Next). + +build_bin_seg_integer(A, Bits, Val, Next) -> + Sz = #k_int{anno=A,val=Bits}, + Seg = #k_literal{anno=A,val=Val}, + #k_bin_seg{anno=A,size=Sz,unit=1,type=integer,flags=[unsigned,big],seg=Seg,next=Next}. + +integer_fits_and_is_expandable(Int, Size) when 0 < Size, Size =< ?EXPAND_MAX_SIZE_SEGMENT -> + case <<Int:Size>> of + <<Int:Size>> -> true; + _ -> false + end; +integer_fits_and_is_expandable(_Int, _Size) -> + false. + +utf8_fits(Utf8) -> + try + Bin = <<Utf8/utf8>>, + Bits = bit_size(Bin), + <<Int:Bits>> = Bin, + {Int, Bits} + catch + _:_ -> error + end. + %% pattern_list([Cexpr], Sub, State) -> {[Kexpr],Sub,State}. pattern_list(Ces, Sub, St) -> @@ -1535,7 +1603,7 @@ maybe_add_warning(Ke, MatchAnno, St) -> get_line([Line|_]) when is_integer(Line) -> Line; get_line([_|T]) -> get_line(T); get_line([]) -> none. - + get_file([{file,File}|_]) -> File; get_file([_|T]) -> get_file(T); get_file([]) -> "no_file". % should not happen @@ -1743,27 +1811,10 @@ do_combine_lit_pat(#k_tuple{anno=A,es=Es0}) -> do_combine_lit_pat(_) -> throw(not_possible). -combine_bin_segs(#k_bin_seg{size=Size0,unit=Unit,type=integer, - flags=[unsigned,big],seg=Seg,next=Next}) -> - #k_literal{val=Size1} = do_combine_lit_pat(Size0), - #k_literal{val=Int} = do_combine_lit_pat(Seg), - Size = Size1 * Unit, - if - 0 < Size, Size < 64 -> - Bin = <<Int:Size>>, - case Bin of - <<Int:Size>> -> - NextBin = combine_bin_segs(Next), - <<Bin/bits,NextBin/bits>>; - _ -> - %% The integer Int does not fit in the segment, - %% thus it will not match. - throw(not_possible) - end; - true -> - %% Avoid creating huge binary literals. - throw(not_possible) - end; +combine_bin_segs(#k_bin_seg{size=#k_int{val=8},unit=1,type=integer, + flags=[unsigned,big],seg=#k_literal{val=Int},next=Next}) + when is_integer(Int), 0 =< Int, Int =< 255 -> + <<Int,(combine_bin_segs(Next))/bits>>; combine_bin_segs(#k_bin_end{}) -> <<>>; combine_bin_segs(_) -> @@ -1833,11 +1884,10 @@ handle_bin_con_not_possible([]) -> []. select_bin_int([#iclause{pats=[#k_bin_seg{anno=A,type=integer, size=#k_int{val=Bits0}=Sz,unit=U, flags=Fl,seg=#k_literal{val=Val}, - next=N}|Ps]}=C|Cs0]) - when is_integer(Val) -> + next=N}|Ps]}=C|Cs0]) -> Bits = U * Bits0, if - Bits > 1024 -> throw(not_possible); %Expands the code too much. + Bits > ?EXPAND_MAX_SIZE_SEGMENT -> throw(not_possible); %Expands the code too much. true -> ok end, select_assert_match_possible(Bits, Val, Fl), @@ -1848,16 +1898,6 @@ select_bin_int([#iclause{pats=[#k_bin_seg{anno=A,type=integer, end, Cs = select_bin_int_1(Cs0, Bits, Fl, Val), [{k_bin_int,[C#iclause{pats=[P|Ps]}|Cs]}]; -select_bin_int([#iclause{pats=[#k_bin_seg{anno=A,type=utf8, - flags=[unsigned,big]=Fl, - seg=#k_literal{val=Val0}, - next=N}|Ps]}=C|Cs0]) - when is_integer(Val0) -> - {Val,Bits} = select_utf8(Val0), - P = #k_bin_int{anno=A,size=#k_int{val=Bits},unit=1, - flags=Fl,val=Val,next=N}, - Cs = select_bin_int_1(Cs0, Bits, Fl, Val), - [{k_bin_int,[C#iclause{pats=[P|Ps]}|Cs]}]; select_bin_int(_) -> throw(not_possible). select_bin_int_1([#iclause{pats=[#k_bin_seg{anno=A,type=integer, @@ -1872,18 +1912,6 @@ select_bin_int_1([#iclause{pats=[#k_bin_seg{anno=A,type=integer, end, P = #k_bin_int{anno=A,size=Sz,unit=U,flags=Fl,val=Val,next=N}, [C#iclause{pats=[P|Ps]}|select_bin_int_1(Cs, Bits, Fl, Val)]; -select_bin_int_1([#iclause{pats=[#k_bin_seg{anno=A,type=utf8, - flags=Fl, - seg=#k_literal{val=Val0}, - next=N}|Ps]}=C|Cs], - Bits, Fl, Val) when is_integer(Val0) -> - case select_utf8(Val0) of - {Val,Bits} -> ok; - {_,_} -> throw(not_possible) - end, - P = #k_bin_int{anno=A,size=#k_int{val=Bits},unit=1, - flags=[unsigned,big],val=Val,next=N}, - [C#iclause{pats=[P|Ps]}|select_bin_int_1(Cs, Bits, Fl, Val)]; select_bin_int_1([], _, _, _) -> []; select_bin_int_1(_, _, _, _) -> throw(not_possible). @@ -1909,17 +1937,6 @@ match_fun(Val) -> {match,Bs} end. -select_utf8(Val0) -> - try - Bin = <<Val0/utf8>>, - Size = bit_size(Bin), - <<Val:Size>> = Bin, - {Val,Size} - catch - error:_ -> - throw(not_possible) - end. - %% match_value([Var], Con, [Clause], Default, State) -> {SelectExpr,State}. %% At this point all the clauses have the same constructor, we must %% now separate them according to value. @@ -2039,7 +2056,8 @@ match_clause([U|Us], [C|_]=Cs0, Def, St0) -> {Match0,Vs,St1} = get_match(get_con(Cs0), St0), Match = sub_size_var(Match0, Cs0), {Cs1,St2} = new_clauses(Cs0, U, St1), - {B,St3} = match(Vs ++ Us, Cs1, Def, St2), + Cs2 = squeeze_clauses_by_bin_integer_count(Cs1, []), + {B,St3} = match(Vs ++ Us, Cs2, Def, St2), {#k_val_clause{anno=Anno,val=Match,body=B},St3}. sub_size_var(#k_bin_seg{size=#k_var{name=Name}=Kvar}=BinSeg, [#iclause{isub=Sub}|_]) -> @@ -2109,6 +2127,102 @@ new_clauses(Cs0, U, St) -> end, Cs0), {Cs1,St}. +%% group and squeeze +%% The goal of those functions is to group subsequent integer k_bin_seg +%% literals by count so we can leverage bs_get_integer_16 whenever possible. +%% +%% The priority is to create large groups. So if we have three clauses matching +%% on 16-bits/16-bits/8-bits, we will first have a single 8-bits match for all +%% three clauses instead of clauses (one with 16 and another with 8). But note +%% the algorithm is recursive, so the remaining 8-bits for the first two clauses +%% will be grouped next. +%% +%% We also try to not create too large groups. If we have too many clauses, +%% it is preferrable to match on 8-bits, select a branch, then match on the +%% next 8-bits, rather than match on 16-bits which would force us to have +%% to select to many values at the same time, which would not be efficient. +%% +%% Another restriction is that we create groups only if the end of the +%% group is a variadic clause or the end of the binary. That's because +%% if we have 16-bits/16-bits/catch-all, breaking it into a 16-bits lookup +%% will make the catch-all more expensive. +%% +%% Clauses are grouped in reverse when squeezing and then flattened and +%% re-reversed at the end. +squeeze_clauses_by_bin_integer_count([Clause | Clauses], Acc) -> + case clause_count_bin_integer_segments(Clause) of + {literal, N} -> squeeze_clauses_by_bin_integer_count(Clauses, N, 1, [Clause], Acc); + _ -> squeeze_clauses_by_bin_integer_count(Clauses, [[Clause] | Acc]) + end; +squeeze_clauses_by_bin_integer_count(_, Acc) -> + flat_reverse(Acc, []). + +squeeze_clauses_by_bin_integer_count([], N, Count, GroupAcc, Acc) -> + Squeezed = squeeze_clauses(GroupAcc, fix_count_without_variadic_segment(N), Count), + flat_reverse([Squeezed | Acc], []); +squeeze_clauses_by_bin_integer_count([#iclause{pats=[#k_bin_end{} | _]} = Clause], N, Count, GroupAcc, Acc) -> + Squeezed = squeeze_clauses(GroupAcc, fix_count_without_variadic_segment(N), Count), + flat_reverse([[Clause | Squeezed] | Acc], []); +squeeze_clauses_by_bin_integer_count([Clause | Clauses], N, Count, GroupAcc, Acc) -> + case clause_count_bin_integer_segments(Clause) of + {literal, NewN} -> + squeeze_clauses_by_bin_integer_count(Clauses, min(N, NewN), Count + 1, [Clause | GroupAcc], Acc); + + {variadic, NewN} when NewN =< N -> + Squeezed = squeeze_clauses(GroupAcc, NewN, Count), + squeeze_clauses_by_bin_integer_count(Clauses, [[Clause | Squeezed] | Acc]); + + _ -> + squeeze_clauses_by_bin_integer_count(Clauses, [[Clause | GroupAcc] | Acc]) + end. + +clause_count_bin_integer_segments(#iclause{pats=[#k_bin_seg{seg=#k_literal{}} = BinSeg | _]}) -> + count_bin_integer_segments(BinSeg, 0); +clause_count_bin_integer_segments(#iclause{pats=[#k_bin_seg{size=#k_int{val=Size},unit=Unit, + type=integer,flags=[unsigned,big], seg=#k_var{}} | _]}) + when ((Size * Unit) rem 8) =:= 0 -> + {variadic, (Size * Unit) div 8}; +clause_count_bin_integer_segments(_) -> + error. + +count_bin_integer_segments(#k_bin_seg{size=#k_int{val=8},unit=1,type=integer,flags=[unsigned,big], + seg=#k_literal{val=Int},next=Next}, Count) when is_integer(Int), 0 =< Int, Int =< 255 -> + count_bin_integer_segments(Next, Count + 1); +count_bin_integer_segments(_, Count) when Count > 0 -> + {literal, Count}; +count_bin_integer_segments(_, _Count) -> + error. + +%% Since 4 bytes in on 32-bits systems are bignums, we convert +%% anything more than 3 into 2 bytes lookup. The goal is to convert +%% any multi-clause segment into 2-byte lookups with a potential +%% 3 byte lookup at the end. +fix_count_without_variadic_segment(N) when N > 3 -> 2; +fix_count_without_variadic_segment(N) -> N. + +%% If we have more than 16 clauses, then it is better +%% to branch multiple times than getting a large integer. +%% We also abort if we have nothing to squeeze. +squeeze_clauses(Clauses, Size, Count) when Count >= 16; Size == 1 -> Clauses; +squeeze_clauses(Clauses, Size, _Count) -> squeeze_clauses(Clauses, Size). + +squeeze_clauses([#iclause{pats=[#k_bin_seg{seg=#k_literal{}} = BinSeg | Pats]} = Clause | Clauses], Size) -> + [Clause#iclause{pats=[squeeze_segments(BinSeg, 0, 0, Size) | Pats]} | + squeeze_clauses(Clauses, Size)]; +squeeze_clauses([], _Size) -> + []. + +squeeze_segments(#k_bin_seg{size=Sz, seg=#k_literal{val=Val}=Lit} = BinSeg, Acc, Size, 1) -> + BinSeg#k_bin_seg{size=Sz#k_int{val=Size + 8}, seg=Lit#k_literal{val=(Acc bsl 8) bor Val}}; +squeeze_segments(#k_bin_seg{seg=#k_literal{val=Val},next=Next}, Acc, Size, Count) -> + squeeze_segments(Next, (Acc bsl 8) bor Val, Size + 8, Count - 1). + +flat_reverse([Head | Tail], Acc) -> flat_reverse(Tail, flat_reverse_1(Head, Acc)); +flat_reverse([], Acc) -> Acc. + +flat_reverse_1([Head | Tail], Acc) -> flat_reverse_1(Tail, [Head | Acc]); +flat_reverse_1([], Acc) -> Acc. + %% build_guard([GuardClause]) -> GuardExpr. build_guard([]) -> fail; @@ -2446,8 +2560,21 @@ uexpr(Lit, {break,Rs0}, St0) -> {#k_put{anno=#k{us=Used,ns=lit_list_vars(Rs),a=get_kanno(Lit)}, arg=Lit,ret=Rs},Used,St1}. -add_local_function(_, #kern{funs=ignore}=St) -> St; -add_local_function(F, #kern{funs=Funs}=St) -> St#kern{funs=[F|Funs]}. +add_local_function(_, #kern{funs=ignore}=St) -> + St; +add_local_function(#k_fdef{func=Name,arity=Arity}=F, #kern{funs=Funs}=St) -> + case is_defined(Name, Arity, Funs) of + false -> + St#kern{funs=[F|Funs]}; + true -> + St + end. + +is_defined(Name, Arity, [#k_fdef{func=Name,arity=Arity}|_]) -> + true; +is_defined(Name, Arity, [#k_fdef{}|T]) -> + is_defined(Name, Arity, T); +is_defined(_, _, []) -> false. %% Make a #k_fdef{}, making sure that the body is always a #k_match{}. make_fdef(Anno, Name, Arity, Vs, #k_match{}=Body) -> diff --git a/lib/compiler/test/Makefile b/lib/compiler/test/Makefile index 7be23fbb93..44cff40128 100644 --- a/lib/compiler/test/Makefile +++ b/lib/compiler/test/Makefile @@ -16,6 +16,7 @@ MODULES= \ beam_reorder_SUITE \ beam_ssa_SUITE \ beam_type_SUITE \ + beam_types_SUITE \ beam_utils_SUITE \ bif_SUITE \ bs_bincomp_SUITE \ @@ -236,6 +237,6 @@ release_tests_spec: make_emakefile $(INSTALL_DATA) $(ERL_DUMMY_FILES) "$(RELSYSDIR)" rm $(ERL_DUMMY_FILES) chmod -R u+w "$(RELSYSDIR)" - @tar cf - *_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -) + @tar cf - *_SUITE_data property_test | (cd "$(RELSYSDIR)"; tar xf -) release_docs_spec: diff --git a/lib/compiler/test/beam_ssa_SUITE.erl b/lib/compiler/test/beam_ssa_SUITE.erl index 3b510f3528..054f86731a 100644 --- a/lib/compiler/test/beam_ssa_SUITE.erl +++ b/lib/compiler/test/beam_ssa_SUITE.erl @@ -200,6 +200,8 @@ recv(_Config) -> %% tricky_recv_6/0 is a compile-time error. tricky_recv_6(), + recv_coverage(), + ok. sync_wait_mon({Pid, Ref}, Timeout) -> @@ -326,7 +328,7 @@ tricky_recv_5() -> end. %% When fixing tricky_recv_5, we introduced a compiler crash when the common -%% exit block was ?BADARG_BLOCK and floats were in the picture. +%% exit block was ?EXCEPTION_BLOCK and floats were in the picture. tricky_recv_6() -> RefA = make_ref(), RefB = make_ref(), @@ -337,6 +339,69 @@ tricky_recv_6() -> ok end. +recv_coverage() -> + self() ! 1, + a = recv_coverage_1(), + self() ! 2, + b = recv_coverage_1(), + + self() ! 1, + a = recv_coverage_2(), + self() ! 2, + b = recv_coverage_2(), + + ok. + +%% Similar to tricky_recv_5/0, but provides test coverage for the #b_switch{} +%% terminator. +recv_coverage_1() -> + receive + X=1 -> + %% Jump to common exit block through #b_switch{list=L} + case id(0) of + 0 -> a; + 1 -> b; + 2 -> c; + 3 -> d + end; + X=2 -> + %% Jump to common exit block through #b_switch{fail=F} + case id(42) of + 0 -> exit(quit); + 1 -> exit(quit); + 2 -> exit(quit); + 3 -> exit(quit); + _ -> b + end + end, + case X of + 1 -> a; + 2 -> b + end. + +%% Similar to recv_coverage_1/0, providing test coverage for #b_br{}. +recv_coverage_2() -> + receive + X=1 -> + A = id(1), + %% Jump to common exit block through #b_br{succ=S}. + if + A =:= 1 -> a; + true -> exit(quit) + end; + X=2 -> + A = id(2), + %% Jump to common exit block through #b_br{fail=F}. + if + A =:= 1 -> exit(quit); + true -> a + end + end, + case X of + 1 -> a; + 2 -> b + end. + maps(_Config) -> {'EXIT',{{badmatch,#{}},_}} = (catch maps_1(any)), ok. @@ -388,48 +453,8 @@ cover_ssa_dead(_Config) -> 40.0 = percentage(4.0, 10.0), 60.0 = percentage(6, 10), - %% Cover '=:=', followed by '=/='. - false = 'cover__=:=__=/='(41), - true = 'cover__=:=__=/='(42), - false = 'cover__=:=__=/='(43), - - %% Cover '<', followed by '=/='. - true = 'cover__<__=/='(41), - false = 'cover__<__=/='(42), - false = 'cover__<__=/='(43), - - %% Cover '=<', followed by '=/='. - true = 'cover__=<__=/='(41), - true = 'cover__=<__=/='(42), - false = 'cover__=<__=/='(43), - - %% Cover '>=', followed by '=/='. - false = 'cover__>=__=/='(41), - true = 'cover__>=__=/='(42), - true = 'cover__>=__=/='(43), - - %% Cover '>', followed by '=/='. - false = 'cover__>__=/='(41), - false = 'cover__>__=/='(42), - true = 'cover__>__=/='(43), - ok. -'cover__=:=__=/='(X) when X =:= 42 -> X =/= 43; -'cover__=:=__=/='(_) -> false. - -'cover__<__=/='(X) when X < 42 -> X =/= 42; -'cover__<__=/='(_) -> false. - -'cover__=<__=/='(X) when X =< 42 -> X =/= 43; -'cover__=<__=/='(_) -> false. - -'cover__>=__=/='(X) when X >= 42 -> X =/= 41; -'cover__>=__=/='(_) -> false. - -'cover__>__=/='(X) when X > 42 -> X =/= 42; -'cover__>__=/='(_) -> false. - format_str(Str, FormatData, IoList, EscChars) -> Escapable = FormatData =:= escapable, case id(Str) of diff --git a/lib/compiler/test/beam_types_SUITE.erl b/lib/compiler/test/beam_types_SUITE.erl new file mode 100644 index 0000000000..8e71a716cd --- /dev/null +++ b/lib/compiler/test/beam_types_SUITE.erl @@ -0,0 +1,124 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(beam_types_SUITE). + +-include_lib("compiler/src/beam_types.hrl"). + +-export([all/0, suite/0, groups/0, + init_per_suite/1, end_per_suite/1]). + +-export([absorption/1, + associativity/1, + commutativity/1, + idempotence/1, + identity/1]). + +-export([binary_absorption/1, + integer_absorption/1, + integer_associativity/1]). + +suite() -> + [{ct_hooks,[ts_install_cth]}]. + +all() -> + [{group,property_tests}, + binary_absorption, + integer_absorption, + integer_associativity]. + +groups() -> + [{property_tests,[parallel], + [absorption, + associativity, + commutativity, + idempotence, + identity]}]. + +init_per_suite(Config) -> + ct_property_test:init_per_suite(Config). + +end_per_suite(Config) -> + Config. + +absorption(Config) when is_list(Config) -> + %% manual test: proper:quickcheck(beam_types_prop:absorption()). + true = ct_property_test:quickcheck(beam_types_prop:absorption(), Config). + +associativity(Config) when is_list(Config) -> + %% manual test: proper:quickcheck(beam_types_prop:associativity()). + true = ct_property_test:quickcheck(beam_types_prop:associativity(), Config). + +commutativity(Config) when is_list(Config) -> + %% manual test: proper:quickcheck(beam_types_prop:commutativity()). + true = ct_property_test:quickcheck(beam_types_prop:commutativity(), Config). + +idempotence(Config) when is_list(Config) -> + %% manual test: proper:quickcheck(beam_types_prop:idempotence()). + true = ct_property_test:quickcheck(beam_types_prop:idempotence(), Config). + +identity(Config) when is_list(Config) -> + %% manual test: proper:quickcheck(beam_types_prop:identity()). + true = ct_property_test:quickcheck(beam_types_prop:identity(), Config). + +binary_absorption(Config) when is_list(Config) -> + %% These binaries should meet into {binary,12} as that's the best common + %% unit for both types. + A = #t_bitstring{unit=4}, + B = #t_bitstring{unit=6}, + + #t_bitstring{unit=12} = beam_types:meet(A, B), + #t_bitstring{unit=2} = beam_types:join(A, B), + + A = beam_types:meet(A, beam_types:join(A, B)), + A = beam_types:join(A, beam_types:meet(A, B)), + + ok. + +integer_absorption(Config) when is_list(Config) -> + %% Integers that don't overlap at all should never meet. + A = #t_integer{elements={2,3}}, + B = #t_integer{elements={4,5}}, + + none = beam_types:meet(A, B), + #t_integer{elements={2,5}} = beam_types:join(A, B), + + A = beam_types:meet(A, beam_types:join(A, B)), + A = beam_types:join(A, beam_types:meet(A, B)), + + ok. + +integer_associativity(Config) when is_list(Config) -> + A = #t_integer{elements={3,5}}, + B = #t_integer{elements={4,6}}, + C = #t_integer{elements={5,5}}, + + %% a ∨ (b ∨ c) = (a ∨ b) ∨ c, + LHS_Join = beam_types:join(A, beam_types:join(B, C)), + RHS_Join = beam_types:join(beam_types:join(A, B), C), + #t_integer{elements={3,6}} = LHS_Join = RHS_Join, + + %% a ∧ (b ∧ c) = (a ∧ b) ∧ c. + LHS_Meet = beam_types:meet(A, beam_types:meet(B, C)), + RHS_Meet = beam_types:meet(beam_types:meet(A, B), C), + #t_integer{elements={5,5}} = LHS_Meet = RHS_Meet, + + ok. + diff --git a/lib/compiler/test/beam_validator_SUITE.erl b/lib/compiler/test/beam_validator_SUITE.erl index 20f6cb2691..e4e34ec0d2 100644 --- a/lib/compiler/test/beam_validator_SUITE.erl +++ b/lib/compiler/test/beam_validator_SUITE.erl @@ -35,7 +35,8 @@ map_field_lists/1,cover_bin_opt/1, val_dsetel/1,bad_tuples/1,bad_try_catch_nesting/1, receive_stacked/1,aliased_types/1,type_conflict/1, - infer_on_eq/1,infer_dead_value/1]). + infer_on_eq/1,infer_dead_value/1,infer_on_ne/1, + branch_to_try_handler/1,call_without_stack/1]). -include_lib("common_test/include/ct.hrl"). @@ -65,7 +66,8 @@ groups() -> map_field_lists,cover_bin_opt,val_dsetel, bad_tuples,bad_try_catch_nesting, receive_stacked,aliased_types,type_conflict, - infer_on_eq,infer_dead_value]}]. + infer_on_eq,infer_dead_value,infer_on_ne, + branch_to_try_handler,call_without_stack]}]. init_per_suite(Config) -> test_lib:recompile(?MODULE), @@ -148,19 +150,34 @@ stack(Config) when is_list(Config) -> call_last(Config) when is_list(Config) -> Errors = do_val(call_last, Config), - [{{t,a,1},{{call_last,1,{f,8},2},9,{allocated,1}}}, + [{{t,a,1}, + {{call_last,1,{f,8},2},9,{allocated,1}}}, {{t,b,1}, - {{call_ext_last,2,{extfunc,lists,seq,2},2}, - 10, - {allocated,1}}}] = Errors, + {{call_ext_last,2,{extfunc,lists,seq,2},2},10,{allocated,1}}}, + {{t,baz,2}, + {{call_ext_only,2,{extfunc,erlang,put,2}},5,{allocated,0}}}, + {{t,biz,2}, + {{call_only,2,{f,10}},5,{allocated,0}}}] = Errors, + ok. + +call_without_stack(Config) when is_list(Config) -> + Errors = do_val(call_without_stack, Config), + [{{t,local,2}, + {{call,2,{f,2}},4,{allocated,none}}}, + {{t,remote,2}, + {{call_ext,2,{extfunc,lists,seq,2}},4,{allocated,none}}}] = Errors, ok. merge_undefined(Config) when is_list(Config) -> Errors = do_val(merge_undefined, Config), - [{{t,handle_call,2}, + [{{t,undecided,2}, {{call_ext,2,{extfunc,debug,filter,2}}, 22, - {uninitialized_reg,{y,_}}}}] = Errors, + {allocated,undecided}}}, + {{t,uninitialized,2}, + {{call_ext,2,{extfunc,io,format,2}}, + 17, + {uninitialized_reg,{y,1}}}}] = Errors, ok. uninit(Config) when is_list(Config) -> @@ -217,11 +234,11 @@ bad_catch_try(Config) when is_list(Config) -> {{catch_end,{x,9}}, 8,{invalid_tag_register,{x,9}}}}, {{bad_catch_try,bad_3,1}, - {{catch_end,{y,1}},9,{invalid_tag,{y,1},{atom,kalle}}}}, + {{catch_end,{y,1}},9,{invalid_tag,{y,1},{t_atom,[kalle]}}}}, {{bad_catch_try,bad_4,1}, {{'try',{x,0},{f,15}},5,{invalid_tag_register,{x,0}}}}, {{bad_catch_try,bad_5,1}, - {{try_case,{y,1}},12,{invalid_tag,{y,1},term}}}, + {{try_case,{y,1}},12,{invalid_tag,{y,1},any}}}, {{bad_catch_try,bad_6,1}, {{move,{integer,1},{y,1}},7, {invalid_store,{y,1}}}}] = Errors, @@ -232,7 +249,7 @@ cons_guard(Config) when is_list(Config) -> [{{cons,foo,1}, {{get_list,{x,0},{x,1},{x,2}}, 5, - {bad_type,{needed,cons},{actual,term}}}}] = Errors, + {bad_type,{needed,cons},{actual,any}}}}] = Errors, ok. freg_range(Config) when is_list(Config) -> @@ -263,7 +280,7 @@ freg_uninit(Config) when is_list(Config) -> {uninitialized_reg,{fr,1}}}}, {{t,sum_2,2}, {{bif,fadd,{f,0},[{fr,0},{fr,1}],{fr,0}}, - 9, + 10, {uninitialized_reg,{fr,0}}}}] = Errors, ok. @@ -520,9 +537,9 @@ bad_tuples(Config) -> {{bad_tuples,long,2}, {{put,{atom,too_long}},8,not_building_a_tuple}}, {{bad_tuples,self_referential,1}, - {{put,{x,1}},7,{tuple_in_progress,{x,1}}}}, + {{put,{x,1}},7,{unfinished_tuple,{x,1}}}}, {{bad_tuples,short,1}, - {{move,{x,1},{x,0}},7,{tuple_in_progress,{x,1}}}}] = Errors, + {{move,{x,1},{x,0}},7,{unfinished_tuple,{x,1}}}}] = Errors, ok. @@ -705,6 +722,25 @@ idv_1({_A, _B, _C, _D, _E, F, G}, idv_1(_A, _B) -> error. +%% ERL-998; type inference for select_val (#b_switch{}) was more clever than +%% that for is_ne_exact (#b_br{}), sometimes failing validation when the type +%% optimization pass acted on the former and the validator got the latter. + +-record(ion, {state}). + +infer_on_ne(Config) when is_list(Config) -> + #ion{state = closing} = ion_1(#ion{ state = id(open) }), + #ion{state = closing} = ion_close(#ion{ state = open }), + ok. + +ion_1(State = #ion{state = open}) -> ion_2(State); +ion_1(State = #ion{state = closing}) -> ion_2(State). + +ion_2(State = #ion{state = open}) -> ion_close(State); +ion_2(#ion{state = closing}) -> ok. + +ion_close(State = #ion{}) -> State#ion{state = closing}. + %% ERL-995: The first solution to ERIERL-348 was incomplete and caused %% validation to fail when living values depended on delayed type inference on %% "dead" values. @@ -722,6 +758,17 @@ idv_2(State) -> idv_called_once(_State) -> ok. +%% Direct jumps to try/catch handlers crash the emulator and must fail +%% validation. This is provoked by OTP-15945. + +branch_to_try_handler(Config) -> + Errors = do_val(branch_to_try_handler, Config), + [{{branch_to_try_handler,main,1}, + {{bif,tuple_size,{f,3},[{y,0}],{x,0}}, + 12, + {illegal_branch,try_handler,3}}}] = Errors, + ok. + %%%------------------------------------------------------------------------- transform_remove(Remove, Module) -> diff --git a/lib/compiler/test/beam_validator_SUITE_data/branch_to_try_handler.S b/lib/compiler/test/beam_validator_SUITE_data/branch_to_try_handler.S new file mode 100644 index 0000000000..6d43ec7b54 --- /dev/null +++ b/lib/compiler/test/beam_validator_SUITE_data/branch_to_try_handler.S @@ -0,0 +1,48 @@ +{module, branch_to_try_handler}. %% version = 0 + +{exports, [{main,1}]}. + +{attributes, []}. + +{labels, 11}. + +{function, main, 1, 2}. + {label,1}. + {line,[{location,"t.erl",4}]}. + {func_info,{atom,branch_to_try_handler},{atom,main},1}. + {label,2}. + {allocate,2,1}. + {move,{x,0},{y,0}}. + {'try',{y,1},{f,3}}. + {move,{atom,ignored},{x,0}}. + {line,[{location,"t.erl",6}]}. + {call,1,{f,6}}. + {'%',{type_info,{x,0},{t_atom,[ignored]}}}. + {line,[{location,"t.erl",7}]}. + %% + %% Fail directly to the try handler instead of throwing an exception; this + %% will crash the emulator. + %% + {bif,tuple_size,{f,3},[{y,0}],{x,0}}. + %% + {test,is_eq_exact,{f,4},[{x,0},{integer,1}]}. + {move,{atom,error},{x,0}}. + {try_end,{y,1}}. + {deallocate,2}. + return. + {label,3}. + {try_case,{y,1}}. + {move,{atom,ok},{x,0}}. + {deallocate,2}. + return. + {label,4}. + {line,[{location,"t.erl",7}]}. + {badmatch,{x,0}}. + +{function, id, 1, 6}. + {label,5}. + {line,[{location,"t.erl",13}]}. + {func_info,{atom,branch_to_try_handler},{atom,id},1}. + {label,6}. + {'%',{type_info,{x,0},{t_atom,[ignored]}}}. + return. diff --git a/lib/compiler/test/beam_validator_SUITE_data/call_last.S b/lib/compiler/test/beam_validator_SUITE_data/call_last.S index 827b6c0ae6..ff81da1b57 100644 --- a/lib/compiler/test/beam_validator_SUITE_data/call_last.S +++ b/lib/compiler/test/beam_validator_SUITE_data/call_last.S @@ -1,6 +1,6 @@ {module, call_last}. %% version = 0 -{exports, [{a,1},{b,1},{bar,1},{foo,1},{module_info,0},{module_info,1}]}. +{exports, [{a,1},{b,1},{bar,1},{foo,1},{baz,2},{biz,2}]}. {attributes, []}. @@ -53,19 +53,16 @@ {'%live',1}. return. - -{function, module_info, 0, 10}. +{function, baz, 2, 10}. {label,9}. - {func_info,{atom,t},{atom,module_info},0}. + {func_info,{atom,t},{atom,baz},2}. {label,10}. - {move,{atom,t},{x,0}}. - {call_ext_only,1,{extfunc,erlang,get_module_info,1}}. - + {allocate,0,2}. + {call_ext_only,2,{extfunc,erlang,put,2}}. -{function, module_info, 1, 12}. +{function, biz, 2, 12}. {label,11}. - {func_info,{atom,t},{atom,module_info},1}. + {func_info,{atom,t},{atom,biz},2}. {label,12}. - {move,{x,0},{x,1}}. - {move,{atom,t},{x,0}}. - {call_ext_only,2,{extfunc,erlang,get_module_info,2}}. + {allocate,0,2}. + {call_only,2,{f,10}}. diff --git a/lib/compiler/test/beam_validator_SUITE_data/call_without_stack.S b/lib/compiler/test/beam_validator_SUITE_data/call_without_stack.S new file mode 100644 index 0000000000..9ccbc163e3 --- /dev/null +++ b/lib/compiler/test/beam_validator_SUITE_data/call_without_stack.S @@ -0,0 +1,21 @@ +{module, call_without_stack}. %% version = 0 + +{exports, [{remote,2},{local,2}]}. + +{attributes, []}. + +{labels, 9}. + +{function, remote, 2, 2}. + {label,1}. + {func_info,{atom,t},{atom,remote},2}. + {label,2}. + {call_ext,2,{extfunc,lists,seq,2}}. + if_end. + +{function, local, 2, 4}. + {label,3}. + {func_info,{atom,t},{atom,local},2}. + {label,4}. + {call,2,{f,2}}. + if_end. diff --git a/lib/compiler/test/beam_validator_SUITE_data/freg_uninit.S b/lib/compiler/test/beam_validator_SUITE_data/freg_uninit.S index 71e833446a..2d4cbc9388 100644 --- a/lib/compiler/test/beam_validator_SUITE_data/freg_uninit.S +++ b/lib/compiler/test/beam_validator_SUITE_data/freg_uninit.S @@ -21,12 +21,14 @@ {label,3}. {func_info,{atom,t},{atom,sum_2},2}. {label,4}. + {allocate,0,2}. {fconv,{x,0},{fr,0}}. {fconv,{x,1},{fr,1}}. fclearerror. {fcheckerror,{f,0}}. {call,2,{f,6}}. {bif,fadd,{f,0},[{fr,0},{fr,1}],{fr,0}}. + {deallocate,0}. return. {function, foo, 2, 6}. diff --git a/lib/compiler/test/beam_validator_SUITE_data/merge_undefined.S b/lib/compiler/test/beam_validator_SUITE_data/merge_undefined.S index aa344807e4..3035471f04 100644 --- a/lib/compiler/test/beam_validator_SUITE_data/merge_undefined.S +++ b/lib/compiler/test/beam_validator_SUITE_data/merge_undefined.S @@ -1,15 +1,14 @@ {module, merge_undefined}. %% version = 0 -{exports, [{bar,2},{foo,1},{handle_call,2},{module_info,0},{module_info,1}]}. +{exports, [{uninitialized,2},{undecided,2}]}. {attributes, []}. {labels, 15}. - -{function, handle_call, 2, 2}. +{function, uninitialized, 2, 2}. {label,1}. - {func_info,{atom,t},{atom,handle_call},2}. + {func_info,{atom,t},{atom,uninitialized},2}. {label,2}. {test,is_atom,{f,1},[{x,0}]}. {select_val,{x,0},{f,1},{list,[{atom,gurka},{f,3},{atom,delete},{f,4}]}}. @@ -21,7 +20,7 @@ {move,{atom,nisse},{x,0}}. {call_ext,1,{extfunc,erlang,exit,1}}. {label,4}. - {allocate_heap,1,6,2}. + {allocate_heap,2,6,2}. {move,{x,1},{y,0}}. {put_list,{integer,112},nil,{x,0}}. {put_list,{integer,126},{x,0},{x,0}}. @@ -51,37 +50,57 @@ {call_ext,1,{extfunc,erlang,exit,1}}. {label,6}. {move,{y,0},{x,0}}. - {call_last,1,{f,8},1}. + {call_last,1,{f,14},1}. - -{function, foo, 1, 8}. +{function, undecided, 2, 8}. {label,7}. - {func_info,{atom,t},{atom,foo},1}. + {func_info,{atom,t},{atom,undecided},2}. {label,8}. - {move,{atom,ok},{x,0}}. - return. - - -{function, bar, 2, 10}. + {test,is_atom,{f,7},[{x,0}]}. + {select_val,{x,0},{f,1},{list,[{atom,gurka},{f,9},{atom,delete},{f,10}]}}. {label,9}. - {func_info,{atom,t},{atom,bar},2}. + {allocate_heap,2,6,2}. + {test,is_eq_exact,{f,11},[{x,0},{atom,ok}]}. + %% This is unreachable since {x,0} is known not to be 'ok'. We should not + %% fail with "uninitialized y registers" on erlang:exit/1 + {move,{atom,nisse},{x,0}}. + {call_ext,1,{extfunc,erlang,exit,1}}. {label,10}. - {move,{atom,ok},{x,0}}. - return. - - -{function, module_info, 0, 12}. + {allocate_heap,1,6,2}. + {move,{x,1},{y,0}}. + {put_list,{integer,112},nil,{x,0}}. + {put_list,{integer,126},{x,0},{x,0}}. + {put_list,{y,0},nil,{x,1}}. + {'%live',2}. + {call_ext,2,{extfunc,io,format,2}}. + {test,is_ne_exact,{f,12},[{x,0},{atom,ok}]}. {label,11}. - {func_info,{atom,t},{atom,module_info},0}. + %% The number of allocated Y registers are in conflict here. + {move,{atom,logReader},{x,1}}. + {move,{atom,console},{x,0}}. + {call_ext,2,{extfunc,debug,filter,2}}. + {test_heap,14,1}. + {put_list,{atom,logReader},nil,{x,1}}. + {put_list,{atom,console},{x,1},{x,1}}. + {put_tuple,3,{x,2}}. + {put,{atom,debug}}. + {put,{atom,filter}}. + {put,{x,1}}. + {put_tuple,2,{x,1}}. + {put,{x,2}}. + {put,{x,0}}. + {put_tuple,2,{x,0}}. + {put,{atom,badmatch}}. + {put,{x,1}}. + {'%live',1}. + {call_ext,1,{extfunc,erlang,exit,1}}. {label,12}. - {move,{atom,t},{x,0}}. - {call_ext_only,1,{extfunc,erlang,get_module_info,1}}. - + {move,{y,0},{x,0}}. + {call_last,1,{f,8},1}. -{function, module_info, 1, 14}. +{function, foo, 1, 14}. {label,13}. - {func_info,{atom,t},{atom,module_info},1}. + {func_info,{atom,t},{atom,foo},1}. {label,14}. - {move,{x,0},{x,1}}. - {move,{atom,t},{x,0}}. - {call_ext_only,2,{extfunc,erlang,get_module_info,2}}. + {move,{atom,ok},{x,0}}. + return. diff --git a/lib/compiler/test/bs_match_SUITE.erl b/lib/compiler/test/bs_match_SUITE.erl index 145a50f4ad..0dc1d64eeb 100644 --- a/lib/compiler/test/bs_match_SUITE.erl +++ b/lib/compiler/test/bs_match_SUITE.erl @@ -24,7 +24,7 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, init_per_testcase/2,end_per_testcase/2, - verify_highest_opcode/1, + verify_highest_opcode/1, expand_and_squeeze/1, size_shadow/1,int_float/1,otp_5269/1,null_fields/1,wiger/1, bin_tail/1,save_restore/1, partitioned_bs_match/1,function_clause/1, @@ -64,7 +64,7 @@ groups() -> [{p,[], [verify_highest_opcode, size_shadow,int_float,otp_5269,null_fields,wiger, - bin_tail,save_restore, + bin_tail,save_restore,expand_and_squeeze, partitioned_bs_match,function_clause,unit, shared_sub_bins,bin_and_float,dec_subidentifiers, skip_optional_tag,decode_integer,wfbm,degenerated_match,bs_sum, @@ -2021,3 +2021,161 @@ do_exceptions_after_match_failure(Other) -> ok. id(I) -> I. + +expand_and_squeeze(Config) when is_list(Config) -> + %% UTF8 literals are expanded and then squeezed into integer16 + [ + {test,bs_get_integer2,_,_,[_,{integer,16}|_],_} + | _ + ] = binary_match_to_asm([ + ?Q("<<$á/utf8,_/binary>>"), + ?Q("<<$é/utf8,_/binary>>") + ]), + + %% Sized integers are expanded and then squeezed into integer16 + [ + {test,bs_get_integer2,_,_,[_,{integer,16}|_],_} + | _ + ] = binary_match_to_asm([ + ?Q("<<0:32,_/binary>>"), + ?Q("<<\"bbbb\",_/binary>>") + ]), + + %% Groups of 8 bits are squeezed into integer16 + [ + {test,bs_get_integer2,_,_,[_,{integer,16}|_],_} + | _ + ] = binary_match_to_asm([ + ?Q("<<\"aaaa\",_/binary>>"), + ?Q("<<\"bbbb\",_/binary>>") + ]), + + %% Groups of 8 bits with empty binary are also squeezed + [ + {test,bs_get_integer2,_,_,[_,{integer,16}|_],_} + | _ + ] = binary_match_to_asm([ + ?Q("<<\"aaaa\",_/binary>>"), + ?Q("<<\"bbbb\",_/binary>>"), + ?Q("<<>>") + ]), + + %% Groups of 8 bits with float lookup are not squeezed + [ + {test,bs_get_integer2,_,_,[_,{integer,8}|_],_} + | _ + ] = binary_match_to_asm([ + ?Q("<<\"aaaa\",_/binary>>"), + ?Q("<<\"bbbb\",_/binary>>"), + ?Q("<<_/float>>") + ]), + + %% Groups of diverse bits go with minimum possible + [ + {test,bs_get_integer2,_,_,[_,{integer,8}|_],_} + | _ + ] = binary_match_to_asm([ + ?Q("<<\"aa\",_/binary>>"), + ?Q("<<\"bb\",_/binary>>"), + ?Q("<<\"c\",_/binary>>") + ]), + + %% Groups of diverse bits go with minimum possible but are recursive... + [ + {test,bs_get_integer2,_,_,[_,{integer,8}|_],_} + | RestDiverse + ] = binary_match_to_asm([ + ?Q("<<\"aaa\",_/binary>>"), + ?Q("<<\"abb\",_/binary>>"), + ?Q("<<\"c\",_/binary>>") + ]), + + %% so we still perform a 16 bits lookup for the remaining + true = lists:any(fun({test,bs_get_integer2,_,_,[_,{integer,16}|_],_}) -> true; + (_) -> false end, RestDiverse), + + %% Large match is kept as is if there is a sized match later + [ + {test,bs_get_integer2,_,_,[_,{integer,64}|_],_} + | _ + ] = binary_match_to_asm([ + ?Q("<<255,255,255,255,255,255,255,255>>"), + ?Q("<<_:64>>") + ]), + + %% Large match is kept as is with large matches before and after + [ + {test,bs_get_integer2,_,_,[_,{integer,32}|_],_} + | _ + ] = binary_match_to_asm([ + ?Q("<<A:32,_:A>>"), + ?Q("<<0:32>>"), + ?Q("<<_:32>>") + ]), + + %% Large match is kept as is with large matches before and after + [ + {test,bs_get_integer2,_,_,[_,{integer,32}|_],_} + | _ + ] = binary_match_to_asm([ + ?Q("<<A:32,_:A>>"), + ?Q("<<0,0,0,0>>"), + ?Q("<<_:32>>") + ]), + + %% Large match is kept as is with smaller but still large matches before and after + [ + {test,bs_get_integer2,_,_,[_,{integer,32}|_],_} + | _ + ] = binary_match_to_asm([ + ?Q("<<A:32, _:A>>"), + ?Q("<<0:64>>"), + ?Q("<<_:32>>") + ]), + + %% There is no squeezing for groups with more than 16 matches + [ + {test,bs_get_integer2,_,_,[_,{integer,8}|_],_} + | _ + ] = binary_match_to_asm([ + ?Q("<<\"aa\", _/binary>>"), + ?Q("<<\"bb\", _/binary>>"), + ?Q("<<\"cc\", _/binary>>"), + ?Q("<<\"dd\", _/binary>>"), + ?Q("<<\"ee\", _/binary>>"), + ?Q("<<\"ff\", _/binary>>"), + ?Q("<<\"gg\", _/binary>>"), + ?Q("<<\"hh\", _/binary>>"), + ?Q("<<\"ii\", _/binary>>"), + ?Q("<<\"jj\", _/binary>>"), + ?Q("<<\"kk\", _/binary>>"), + ?Q("<<\"ll\", _/binary>>"), + ?Q("<<\"mm\", _/binary>>"), + ?Q("<<\"nn\", _/binary>>"), + ?Q("<<\"oo\", _/binary>>"), + ?Q("<<\"pp\", _/binary>>") + ]), + + ok. + +binary_match_to_asm(Matches) -> + Clauses = [ + begin + Ann = element(2, Match), + {clause,Ann,[Match],[],[{integer,Ann,Return}]} + end || {Match,Return} <- lists:zip(Matches, lists:seq(1, length(Matches))) + ], + + Module = [ + {attribute,1,module,match_to_asm}, + {attribute,2,export,[{example,1}]}, + {function,3,example,1,Clauses} + ], + + {ok,match_to_asm,{match_to_asm,_Exports,_Attrs,Funs,_},_} = + compile:forms(Module, [return, to_asm]), + + [{function,example,1,2,AllInstructions}|_] = Funs, + [{label,_},{line,_},{func_info,_,_,_},{label,_},{'%',_}, + {test,bs_start_match3,_,_,_,_},{bs_get_position,_,_,_}|Instructions] = AllInstructions, + Instructions. diff --git a/lib/compiler/test/compile_SUITE.erl b/lib/compiler/test/compile_SUITE.erl index 53627b9d81..453debc0c1 100644 --- a/lib/compiler/test/compile_SUITE.erl +++ b/lib/compiler/test/compile_SUITE.erl @@ -378,7 +378,6 @@ do_file_listings(DataDir, PrivDir, [File|Files]) -> {dprecg, ".precodegen"}, {dcg, ".codegen"}, {dblk, ".block"}, - {dexcept, ".except"}, {djmp, ".jump"}, {dclean, ".clean"}, {dpeep, ".peep"}, @@ -1383,27 +1382,33 @@ env_compiler_options(_Config) -> bc_options(Config) -> DataDir = proplists:get_value(data_dir, Config), - L = [{101, small_float, [no_get_hd_tl,no_line_info]}, - {103, big, [no_put_tuple2,no_get_hd_tl,no_ssa_opt_record, + L = [{101, small_float, [no_shared_fun_wrappers, + no_get_hd_tl,no_line_info]}, + {103, big, [no_shared_fun_wrappers, + no_put_tuple2,no_get_hd_tl,no_ssa_opt_record, no_line_info,no_stack_trimming]}, - {125, small_float, [no_get_hd_tl,no_line_info,no_ssa_opt_float]}, + {125, small_float, [no_shared_fun_wrappers,no_get_hd_tl, + no_line_info, + no_ssa_opt_float]}, - {132, small, [no_put_tuple2,no_get_hd_tl,no_ssa_opt_record, + {132, small, [no_shared_fun_wrappers, + no_put_tuple2,no_get_hd_tl,no_ssa_opt_record, no_ssa_opt_float,no_line_info,no_bsm3]}, + {136, big, [no_shared_fun_wrappers,no_put_tuple2,no_get_hd_tl, + no_ssa_opt_record,no_line_info]}, + {153, small, [r20]}, {153, small, [r21]}, - {136, big, [no_put_tuple2,no_get_hd_tl, - no_ssa_opt_record,no_line_info]}, - - {153, big, [no_put_tuple2,no_get_hd_tl, no_ssa_opt_record]}, + {153, big, [no_shared_fun_wrappers, + no_put_tuple2,no_get_hd_tl, no_ssa_opt_record]}, {153, big, [r16]}, {153, big, [r17]}, {153, big, [r18]}, {153, big, [r19]}, {153, small_float, [r16]}, - {153, small_float, []}, + {153, small_float, [no_shared_fun_wrappers]}, {158, small_maps, [r17]}, {158, small_maps, [r18]}, @@ -1411,8 +1416,17 @@ bc_options(Config) -> {158, small_maps, [r20]}, {158, small_maps, [r21]}, - {164, small_maps, []}, - {164, big, []} + {164, small_maps, [r22]}, + {164, big, [r22]}, + {164, small_maps, [no_shared_fun_wrappers]}, + {164, big, [no_shared_fun_wrappers]}, + + {168, small, [r22]}, + {168, small, [no_shared_fun_wrappers]}, + + {169, small_maps, []}, + {169, big, []}, + {169, small, []} ], Test = fun({Expected,Mod,Options}) -> diff --git a/lib/compiler/test/guard_SUITE.erl b/lib/compiler/test/guard_SUITE.erl index cea7a374cd..d3d62b53f5 100644 --- a/lib/compiler/test/guard_SUITE.erl +++ b/lib/compiler/test/guard_SUITE.erl @@ -19,7 +19,7 @@ %% -module(guard_SUITE). --include_lib("common_test/include/ct.hrl"). +-include_lib("syntax_tools/include/merl.hrl"). -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, @@ -31,7 +31,8 @@ old_guard_tests/1,complex_guard/1, build_in_guard/1,gbif/1, t_is_boolean/1,is_function_2/1, - tricky/1,rel_ops/1,rel_op_combinations/1,literal_type_tests/1, + tricky/1,rel_ops/1,rel_op_combinations/1, + generated_combinations/1,literal_type_tests/1, basic_andalso_orelse/1,traverse_dcd/1, check_qlc_hrl/1,andalso_semi/1,t_tuple_size/1,binary_part/1, bad_constants/1,bad_guards/1, @@ -51,7 +52,7 @@ groups() -> more_xor_guards,build_in_guard, old_guard_tests,complex_guard,gbif, t_is_boolean,is_function_2,tricky, - rel_ops,rel_op_combinations, + rel_ops,rel_op_combinations,generated_combinations, literal_type_tests,basic_andalso_orelse,traverse_dcd, check_qlc_hrl,andalso_semi,t_tuple_size,binary_part, bad_constants,bad_guards,guard_in_catch,beam_bool_SUITE, @@ -1579,6 +1580,122 @@ redundant_12(X) when X >= 50, X =< 80 -> 2*X; redundant_12(X) when X < 51 -> 5*X; redundant_12(_) -> none. +generated_combinations(Config) -> + case ?MODULE of + guard_SUITE -> generated_combinations_1(Config); + _ -> {skip,"Enough to run this case once."} + end. + +%% Exhaustively test all combinations of relational operators +%% to ensure the correctness of the optimizations in beam_ssa_dead. + +generated_combinations_1(Config) -> + Mod = ?FUNCTION_NAME, + RelOps = ['=:=','=/=','==','/=','<','=<','>=','>'], + Combinations0 = [{Op1,Op2} || Op1 <- RelOps, Op2 <- RelOps], + Combinations1 = gen_lit_combs(Combinations0), + Combinations2 = [{neq,Comb} || + {_Op1,_Lit1,Op2,_Lit2}=Comb <- Combinations1, + Op2 =:= '=/=' orelse Op2 =:= '/='] ++ Combinations1, + Combinations = gen_func_names(Combinations2, 0), + Fs = gen_rel_op_functions(Combinations), + Tree = ?Q(["-module('@Mod@').", + "-compile([export_all,nowarn_export_all])."]) ++ Fs, + %%merl:print(Tree), + Opts = test_lib:opt_opts(?MODULE), + {ok,_Bin} = merl:compile_and_load(Tree, Opts), + test_combinations(Combinations, Mod). + +gen_lit_combs([{Op1,Op2}|T]) -> + [{Op1,7,Op2,6}, + {Op1,7.0,Op2,6}, + {Op1,7,Op2,6.0}, + {Op1,7.0,Op2,6.0}, + + {Op1,7,Op2,7}, + {Op1,7.0,Op2,7}, + {Op1,7,Op2,7.0}, + {Op1,7.0,Op2,7.0}, + + {Op1,6,Op2,7}, + {Op1,6.0,Op2,7}, + {Op1,6,Op2,7.0}, + {Op1,6.0,Op2,7.0}|gen_lit_combs(T)]; +gen_lit_combs([]) -> []. + +gen_func_names([E|Es], I) -> + Name = list_to_atom("f" ++ integer_to_list(I)), + [{Name,E}|gen_func_names(Es, I+1)]; +gen_func_names([], _) -> []. + +gen_rel_op_functions([{Name,{neq,{Op1,Lit1,Op2,Lit2}}}|T]) -> + %% Note that in the translation to SSA, '=/=' will be + %% translated to '=:=' in a guard (with switched success + %% and failure labels). Therefore, to test the optimization, + %% we must use '=/=' (or '/=') in a body context. + %% + %% Here is an example of a generated function: + %% + %% f160(A) when erlang:'>='(A, 7) -> + %% one; + %% f160(A) -> + %% true = erlang:'/='(A, 7), + %% two. + [?Q("'@Name@'(A) when erlang:'@Op1@'(A, _@Lit1@) -> one; + '@Name@'(A) -> true = erlang:'@Op2@'(A, _@Lit2@), two. ")| + gen_rel_op_functions(T)]; +gen_rel_op_functions([{Name,{Op1,Lit1,Op2,Lit2}}|T]) -> + %% Example of a generated function: + %% + %% f721(A) when erlang:'=<'(A, 7.0) -> one; + %% f721(A) when erlang:'<'(A, 6) -> two; + %% f721(_) -> three. + [?Q("'@Name@'(A) when erlang:'@Op1@'(A, _@Lit1@) -> one; + '@Name@'(A) when erlang:'@Op2@'(A, _@Lit2@) -> two; + '@Name@'(_) -> three.")|gen_rel_op_functions(T)]; +gen_rel_op_functions([]) -> []. + +test_combinations([{Name,E}|T], Mod) -> + try + test_combinations_1([5,6,7,8,9], E, fun Mod:Name/1), + test_combination(6.5, E, fun Mod:Name/1) + catch + error:Reason:Stk -> + io:format("~p: ~p\n", [Name,E]), + erlang:raise(error, Reason, Stk) + end, + test_combinations(T, Mod); +test_combinations([], _Mod) -> ok. + +test_combinations_1([V|Vs], E, Fun) -> + test_combination(V, E, Fun), + test_combination(float(V), E, Fun), + test_combinations_1(Vs, E, Fun); +test_combinations_1([], _, _) -> ok. + +test_combination(Val, {neq,Expr}, Fun) -> + Result = eval_combination_expr(Expr, Val), + Result = try + Fun(Val) %Returns 'one' or 'two'. + catch + error:{badmatch,_} -> + three + end; +test_combination(Val, Expr, Fun) -> + Result = eval_combination_expr(Expr, Val), + Result = Fun(Val). + +eval_combination_expr({Op1,Lit1,Op2,Lit2}, Val) -> + case erlang:Op1(Val, Lit1) of + true -> + one; + false -> + case erlang:Op2(Val, Lit2) of + true -> two; + false -> three + end + end. + %% Test type tests on literal values. (From emulator test suites.) literal_type_tests(Config) when is_list(Config) -> case ?MODULE of diff --git a/lib/compiler/test/misc_SUITE.erl b/lib/compiler/test/misc_SUITE.erl index a0b415ceaa..20fadc4fdb 100644 --- a/lib/compiler/test/misc_SUITE.erl +++ b/lib/compiler/test/misc_SUITE.erl @@ -227,15 +227,6 @@ silly_coverage(Config) when is_list(Config) -> {label,2}|non_proper_list]}],99}, expect_error(fun() -> beam_block:module(BlockInput, []) end), - %% beam_except - ExceptInput = {?MODULE,[{foo,0}],[], - [{function,foo,0,2, - [{label,1}, - {line,loc}, - {func_info,{atom,?MODULE},{atom,foo},0}, - {label,2}|non_proper_list]}],99}, - expect_error(fun() -> beam_except:module(ExceptInput, []) end), - %% beam_jump JumpInput = BlockInput, expect_error(fun() -> beam_jump:module(JumpInput, []) end), @@ -283,14 +274,34 @@ silly_coverage(Config) when is_list(Config) -> bad_ssa_lint_input() -> {b_module,#{},t, - [{foobar,1},{module_info,0},{module_info,1}], + [{a,1},{b,1},{c,1},{module_info,0},{module_info,1}], [], [{b_function, - #{func_info => {t,foobar,1},location => {"t.erl",4}}, + #{func_info => {t,a,1},location => {"t.erl",4}}, [{b_var,0}], #{0 => {b_blk,#{},[],{b_ret,#{},{b_var,'@undefined_var'}}}}, 3}, {b_function, + #{func_info => {t,b,1},location => {"t.erl",5}}, + [{b_var,0}], + #{0 => + {b_blk,#{}, + [{b_set,#{},{b_var,'@first_var'},first_op,[]}, + {b_set,#{},{b_var,'@second_var'},second_op,[]}, + {b_set,#{},{b_var,'@ret'},succeeded,[{b_var,'@first_var'}]}], + {b_ret,#{},{b_var,'@ret'}}}}, + 3}, + {b_function, + #{func_info => {t,c,1},location => {"t.erl",6}}, + [{b_var,0}], + #{0 => + {b_blk,#{}, + [{b_set,#{},{b_var,'@first_var'},first_op,[]}, + {b_set,#{},{b_var,'@ret'},succeeded,[{b_var,'@first_var'}]}, + {b_set,#{},{b_var,'@second_var'},second_op,[]}], + {b_ret,#{},{b_var,'@ret'}}}}, + 3}, + {b_function, #{func_info => {t,module_info,0}}, [], #{0 => diff --git a/lib/compiler/test/property_test/beam_types_prop.erl b/lib/compiler/test/property_test/beam_types_prop.erl new file mode 100644 index 0000000000..8199d1bd5a --- /dev/null +++ b/lib/compiler/test/property_test/beam_types_prop.erl @@ -0,0 +1,228 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(beam_types_prop). + +-compile([export_all, nowarn_export_all]). + +%% This module only supports proper, as we don't have an eqc license to test +%% with. + +-proptest([proper]). + +-ifdef(PROPER). + +-include_lib("compiler/src/beam_types.hrl"). + +-include_lib("proper/include/proper.hrl"). +-define(MOD_eqc,proper). + +%% The default repetitions of 100 is a bit too low to reliably cover all type +%% combinations, so we crank it up a bit. +-define(REPETITIONS, 1000). + +absorption() -> + numtests(?REPETITIONS, absorption_1()). + +absorption_1() -> + ?FORALL({TypeA, TypeB}, + ?LET(TypeA, type(), + ?LET(TypeB, type(), {TypeA, TypeB})), + absorption_check(TypeA, TypeB)). + +absorption_check(A, B) -> + %% a ∨ (a ∧ b) = a, + A = join(A, meet(A, B)), + + %% a ∧ (a ∨ b) = a. + A = meet(A, join(A, B)), + + true. + +associativity() -> + numtests(?REPETITIONS, associativity_1()). + +associativity_1() -> + ?FORALL({TypeA, TypeB, TypeC}, + ?LET(TypeA, type(), + ?LET(TypeB, type(), + ?LET(TypeC, type(), {TypeA, TypeB, TypeC}))), + associativity_check(TypeA, TypeB, TypeC)). + +associativity_check(A, B, C) -> + %% a ∨ (b ∨ c) = (a ∨ b) ∨ c, + LHS_Join = join(A, join(B, C)), + RHS_Join = join(join(A, B), C), + LHS_Join = RHS_Join, + + %% a ∧ (b ∧ c) = (a ∧ b) ∧ c. + LHS_Meet = meet(A, meet(B, C)), + RHS_Meet = meet(meet(A, B), C), + LHS_Meet = RHS_Meet, + + true. + +commutativity() -> + numtests(?REPETITIONS, commutativity_1()). + +commutativity_1() -> + ?FORALL({TypeA, TypeB}, + ?LET(TypeA, type(), + ?LET(TypeB, type(), {TypeA, TypeB})), + commutativity_check(TypeA, TypeB)). + +commutativity_check(A, B) -> + %% a ∨ b = b ∨ a, + true = join(A, B) =:= join(B, A), + + %% a ∧ b = b ∧ a. + true = meet(A, B) =:= meet(B, A), + + true. + +idempotence() -> + numtests(?REPETITIONS, idempotence_1()). + +idempotence_1() -> + ?FORALL(Type, type(), idempotence_check(Type)). + +idempotence_check(Type) -> + %% a ∨ a = a, + Type = join(Type, Type), + + %% a ∧ a = a. + Type = meet(Type, Type), + + true. + +identity() -> + ?FORALL(Type, type(), identity_check(Type)). + +identity_check(Type) -> + %% a ∨ [bottom element] = a, + Type = join(Type, none), + + %% a ∧ [top element] = a. + Type = meet(Type, any), + + true. + +meet(A, B) -> beam_types:meet(A, B). +join(A, B) -> beam_types:join(A, B). + +%%% +%%% Generators +%%% + +type() -> + type(0). + +type(Depth) -> + oneof(nested_types(Depth) ++ + numerical_types() ++ + list_types() ++ + other_types()). + +other_types() -> + [any, + gen_atom(), + gen_binary(), + none]. + +list_types() -> + [cons, list, nil]. + +numerical_types() -> + [gen_integer(), float, number]. + +nested_types(Depth) when Depth >= 3 -> [none]; +nested_types(Depth) -> [#t_map{}, gen_union(Depth + 1), gen_tuple(Depth + 1)]. + +gen_atom() -> + ?LET(Size, range(0, ?ATOM_SET_SIZE), + case Size of + 0 -> + #t_atom{}; + _ -> + ?LET(Set, sized_list(Size, gen_atom_val()), + begin + #t_atom{elements=ordsets:from_list(Set)} + end) + end). + +gen_atom_val() -> + ?LET(N, range($0, $~), list_to_atom([N])). + +gen_binary() -> + ?SHRINK(#t_bitstring{unit=range(1, 128)}, [#t_bitstring{unit=1}]). + +gen_integer() -> + oneof([gen_integer_bounded(), #t_integer{}]). + +gen_integer_bounded() -> + ?LET({A, B}, {integer(), integer()}, + begin + #t_integer{elements={min(A,B), max(A,B)}} + end). + +gen_tuple(Depth) -> + ?SIZED(Size, + ?LET({Exact, Elements}, {boolean(), gen_tuple_elements(Size, Depth)}, + begin + #t_tuple{exact=Exact, + size=Size, + elements=Elements} + end)). + +gen_union(Depth) -> + ?LAZY(oneof([gen_wide_union(Depth), gen_tuple_union(Depth)])). + +gen_wide_union(Depth) -> + ?LET({A, B, C, D}, {oneof(nested_types(Depth)), + oneof(numerical_types()), + oneof(list_types()), + oneof(other_types())}, + begin + T0 = join(A, B), + T1 = join(T0, C), + join(T1, D) + end). + +gen_tuple_union(Depth) -> + ?SIZED(Size, + ?LET(Tuples, sized_list(Size, gen_tuple(Depth)), + lists:foldl(fun join/2, none, Tuples))). + +gen_tuple_elements(Size, Depth) -> + ?LET(Types, sized_list(rand:uniform(Size div 4 + 1), gen_element(Depth)), + maps:from_list([{rand:uniform(Size), T} || T <- Types])). + +gen_element(Depth) -> + ?LAZY(?SUCHTHAT(Type, type(Depth), + case Type of + any -> false; + none -> false; + _ -> true + end)). + +sized_list(0, _Gen) -> []; +sized_list(N, Gen) -> [Gen | sized_list(N - 1, Gen)]. + +-endif. diff --git a/lib/compiler/test/test_lib.erl b/lib/compiler/test/test_lib.erl index 34410e4b2a..4b68e663cd 100644 --- a/lib/compiler/test/test_lib.erl +++ b/lib/compiler/test/test_lib.erl @@ -79,9 +79,11 @@ opt_opts(Mod) -> (no_put_tuple2) -> true; (no_recv_opt) -> true; (no_share_opt) -> true; + (no_shared_fun_wrappers) -> true; (no_ssa_float) -> true; (no_ssa_opt) -> true; (no_stack_trimming) -> true; + (no_swap) -> true; (no_type_opt) -> true; (_) -> false end, Opts). diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index afe56aa7d6..e5812bee15 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -38,3 +38,4 @@ SPECIAL_TARGETS = include $(ERL_TOP)/make/otp_subdir.mk +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/crypto/c_src/atoms.c b/lib/crypto/c_src/atoms.c index bbeb329fa2..fc983ec03b 100644 --- a/lib/crypto/c_src/atoms.c +++ b/lib/crypto/c_src/atoms.c @@ -89,6 +89,8 @@ ERL_NIF_TERM atom_ecdsa; #ifdef HAVE_ED_CURVE_DH ERL_NIF_TERM atom_x25519; ERL_NIF_TERM atom_x448; +ERL_NIF_TERM atom_ed25519; +ERL_NIF_TERM atom_ed448; #endif ERL_NIF_TERM atom_eddsa; @@ -219,6 +221,8 @@ int init_atoms(ErlNifEnv *env, const ERL_NIF_TERM fips_mode, const ERL_NIF_TERM #ifdef HAVE_ED_CURVE_DH atom_x25519 = enif_make_atom(env,"x25519"); atom_x448 = enif_make_atom(env,"x448"); + atom_ed25519 = enif_make_atom(env,"ed25519"); + atom_ed448 = enif_make_atom(env,"ed448"); #endif atom_eddsa = enif_make_atom(env,"eddsa"); #ifdef HAVE_EDDSA diff --git a/lib/crypto/c_src/atoms.h b/lib/crypto/c_src/atoms.h index 0e2f1a0022..956aab93eb 100644 --- a/lib/crypto/c_src/atoms.h +++ b/lib/crypto/c_src/atoms.h @@ -93,6 +93,8 @@ extern ERL_NIF_TERM atom_ecdsa; #ifdef HAVE_ED_CURVE_DH extern ERL_NIF_TERM atom_x25519; extern ERL_NIF_TERM atom_x448; +extern ERL_NIF_TERM atom_ed25519; +extern ERL_NIF_TERM atom_ed448; #endif extern ERL_NIF_TERM atom_eddsa; diff --git a/lib/crypto/c_src/crypto.c b/lib/crypto/c_src/crypto.c index 802818541b..2ea7b7c329 100644 --- a/lib/crypto/c_src/crypto.c +++ b/lib/crypto/c_src/crypto.c @@ -95,7 +95,7 @@ static ErlNifFunc nif_funcs[] = { {"dh_generate_key_nif", 4, dh_generate_key_nif, 0}, {"dh_compute_key_nif", 3, dh_compute_key_nif, 0}, {"evp_compute_key_nif", 3, evp_compute_key_nif, 0}, - {"evp_generate_key_nif", 1, evp_generate_key_nif, 0}, + {"evp_generate_key_nif", 2, evp_generate_key_nif, 0}, {"privkey_to_pubkey_nif", 2, privkey_to_pubkey_nif, 0}, {"srp_value_B_nif", 5, srp_value_B_nif, 0}, {"srp_user_secret_nif", 7, srp_user_secret_nif, 0}, diff --git a/lib/crypto/c_src/evp.c b/lib/crypto/c_src/evp.c index 3bf66bfffe..fb6495a640 100644 --- a/lib/crypto/c_src/evp.c +++ b/lib/crypto/c_src/evp.c @@ -106,25 +106,34 @@ ERL_NIF_TERM evp_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM a EVP_PKEY_CTX *ctx = NULL; EVP_PKEY *pkey = NULL; ERL_NIF_TERM ret_pub, ret_prv, ret; + ErlNifBinary prv_key; size_t key_len; unsigned char *out_pub = NULL, *out_priv = NULL; - ASSERT(argc == 1); - if (argv[0] == atom_x25519) type = EVP_PKEY_X25519; else if (argv[0] == atom_x448) type = EVP_PKEY_X448; + else if (argv[0] == atom_ed25519) + type = EVP_PKEY_ED25519; + else if (argv[0] == atom_ed448) + type = EVP_PKEY_ED448; else goto bad_arg; - if ((ctx = EVP_PKEY_CTX_new_id(type, NULL)) == NULL) - goto bad_arg; - - if (EVP_PKEY_keygen_init(ctx) != 1) - goto err; - if (EVP_PKEY_keygen(ctx, &pkey) != 1) - goto err; + if (argv[1] == atom_undefined) { + if ((ctx = EVP_PKEY_CTX_new_id(type, NULL)) == NULL) + goto bad_arg; + if (EVP_PKEY_keygen_init(ctx) != 1) + goto err; + if (EVP_PKEY_keygen(ctx, &pkey) != 1) + goto err; + } else { + if (!enif_inspect_binary(env, argv[1], &prv_key)) + goto bad_arg; + if ((pkey = EVP_PKEY_new_raw_private_key(type, NULL, prv_key.data, prv_key.size)) == NULL) + goto bad_arg; + } if (EVP_PKEY_get_raw_public_key(pkey, NULL, &key_len) != 1) goto err; diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl index 965697578d..1c32895dbf 100644 --- a/lib/crypto/src/crypto.erl +++ b/lib/crypto/src/crypto.erl @@ -1761,21 +1761,21 @@ pkey_crypt_nif(_Algorithm, _In, _Key, _Options, _IsPrivate, _IsEncrypt) -> ?nif_ -spec generate_key(Type, Params) -> {PublicKey, PrivKeyOut} - when Type :: dh | ecdh | rsa | srp, + when Type :: dh | ecdh | eddsa | rsa | srp, PublicKey :: dh_public() | ecdh_public() | rsa_public() | srp_public(), PrivKeyOut :: dh_private() | ecdh_private() | rsa_private() | {srp_public(),srp_private()}, - Params :: dh_params() | ecdh_params() | rsa_params() | srp_gen_params() + Params :: dh_params() | ecdh_params() | eddsa_params() | rsa_params() | srp_gen_params() . generate_key(Type, Params) -> generate_key(Type, Params, undefined). -spec generate_key(Type, Params, PrivKeyIn) -> {PublicKey, PrivKeyOut} - when Type :: dh | ecdh | rsa | srp, + when Type :: dh | ecdh | eddsa | rsa | srp, PublicKey :: dh_public() | ecdh_public() | rsa_public() | srp_public(), PrivKeyIn :: undefined | dh_private() | ecdh_private() | rsa_private() | {srp_public(),srp_private()}, PrivKeyOut :: dh_private() | ecdh_private() | rsa_private() | {srp_public(),srp_private()}, - Params :: dh_params() | ecdh_params() | rsa_params() | srp_comp_params() + Params :: dh_params() | ecdh_params() | eddsa_params() | rsa_params() | srp_comp_params() . generate_key(dh, DHParameters0, PrivateKey) -> @@ -1813,15 +1813,17 @@ generate_key(rsa, {ModulusSize, PublicExponent}, undefined) -> {lists:sublist(Private, 2), Private} end; - -generate_key(ecdh, Curve, undefined) when Curve == x448 ; - Curve == x25519 -> - evp_generate_key_nif(Curve); +generate_key(ecdh, Curve, PrivKey) when Curve == x448 ; + Curve == x25519 -> + evp_generate_key_nif(Curve, ensure_int_as_bin(PrivKey)); generate_key(ecdh, Curve, PrivKey) -> - ec_key_generate(nif_curve_params(Curve), ensure_int_as_bin(PrivKey)). + ec_key_generate(nif_curve_params(Curve), ensure_int_as_bin(PrivKey)); +generate_key(eddsa, Curve, PrivKey) when Curve == ed448 ; + Curve == ed25519 -> + evp_generate_key_nif(Curve, ensure_int_as_bin(PrivKey)). -evp_generate_key_nif(_Curve) -> ?nif_stub. +evp_generate_key_nif(_Curve, _PrivKey) -> ?nif_stub. -spec compute_key(Type, OthersPublicKey, MyPrivateKey, Params) diff --git a/lib/crypto/test/crypto_SUITE.erl b/lib/crypto/test/crypto_SUITE.erl index 0da70d5592..614d14029b 100644 --- a/lib/crypto/test/crypto_SUITE.erl +++ b/lib/crypto/test/crypto_SUITE.erl @@ -202,11 +202,13 @@ groups() -> {ecdsa, [], [sign_verify %% Does not work yet: ,public_encrypt, private_encrypt ]}, - {ed25519, [], [sign_verify + {ed25519, [], [sign_verify, %% Does not work yet: ,public_encrypt, private_encrypt + generate ]}, - {ed448, [], [sign_verify + {ed448, [], [sign_verify, %% Does not work yet: ,public_encrypt, private_encrypt + generate ]}, {dh, [], [generate_compute, compute_bug]}, {ecdh, [], [use_all_elliptic_curves, compute, generate]}, @@ -1429,7 +1431,7 @@ do_compute({ecdh = Type, Pub, Priv, Curve, SharedSecret}) -> ct:fail({{crypto, compute_key, [Type, Pub, Priv, Curve]}, {expected, SharedSecret}, {got, Other}}) end. -do_generate({ecdh = Type, Curve, Priv, Pub}) -> +do_generate({Type, Curve, Priv, Pub}) when Type == ecdh ; Type == eddsa -> case crypto:generate_key(Type, Curve, Priv) of {Pub, _} -> ok; @@ -1854,7 +1856,10 @@ group_config(ecdsa = Type, Config) -> [{sign_verify, SignVerify}, {pub_priv_encrypt, PubPrivEnc} | Config]; group_config(Type, Config) when Type == ed25519 ; Type == ed448 -> TestVectors = eddsa(Type), - [{sign_verify,TestVectors} | Config]; + Generate = lists:map(fun({Curve, _Hash, Priv, Pub, _Msg, _Signature}) -> + {eddsa, Curve, Priv, Pub} + end, TestVectors), + [{sign_verify,TestVectors}, {generate, Generate} | Config]; group_config(srp, Config) -> GenerateCompute = [srp3(), srp6(), srp6a(), srp6a_smaller_prime()], [{generate_compute, GenerateCompute} | Config]; @@ -3351,7 +3356,7 @@ srp(ClientPrivate, Generator, Prime, Version, Verifier, ServerPublic, ServerPriv eddsa(ed25519) -> %% https://tools.ietf.org/html/rfc8032#section-7.1 - %% {ALGORITHM, (SHA)}, SECRET KEY, PUBLIC KEY, MESSAGE, SIGNATURE} + %% {ALGORITHM, (SHA), SECRET KEY, PUBLIC KEY, MESSAGE, SIGNATURE} [ %% TEST 1 {ed25519, undefined, @@ -3917,12 +3922,31 @@ ecc() -> "782C37E372BA4520AA62E0FED121D49EF3B543660CFD05FD")}, {ecdh,secp192r1,4, hexstr2point("35433907297CC378B0015703374729D7A4FE46647084E4BA", - "A2649984F2135C301EA3ACB0776CD4F125389B311DB3BE32")}], + "A2649984F2135C301EA3ACB0776CD4F125389B311DB3BE32")}, + %% RFC 7748, 6.2 + {ecdh, x448, + hexstr2bin("9a8f4925d1519f5775cf46b04b5800d4ee9ee8bae8bc5565d498c28d" + "d9c9baf574a9419744897391006382a6f127ab1d9ac2d8c0a598726b"), + hexstr2bin("9b08f7cc31b7e3e67d22d5aea121074a273bd2b83de09c63faa73d2c" + "22c5d9bbc836647241d953d40c5b12da88120d53177f80e532c41fa0")}, + {ecdh, x448, + hexstr2bin("1c306a7ac2a0e2e0990b294470cba339e6453772b075811d8fad0d1d" + "6927c120bb5ee8972b0d3e21374c9c921b09d1b0366f10b65173992d"), + hexstr2bin("3eb7a829b0cd20f5bcfc0b599b6feccf6da4627107bdb0d4f345b430" + "27d8b972fc3e34fb4232a13ca706dcb57aec3dae07bdc1c67bf33609")}, + %% RFC 7748, 6.1 + {ecdh, x25519, + hexstr2bin("77076d0a7318a57d3c16c17251b26645df4c2f87ebc0992ab177fba51db92c2a"), + hexstr2bin("8520f0098930a754748b7ddcb43ef75a0dbf3a0d26381af4eba4a98eaa9b4e6a")}, + {ecdh, x25519, + hexstr2bin("5dab087e624a8a4b79e17f8b83800ee66f3bb1292618b6fd1c2f8b27ff88e0eb"), + hexstr2bin("de9edb7d7b7dc1b4d35b61c2ece435373f8343c85b78674dadfc7e146f882b4f")}], lists:filter(fun ({_Type, Curve, _Priv, _Pub}) -> lists:member(Curve, Curves) end, TestCases). + int_to_bin(X) when X < 0 -> int_to_bin_neg(X, []); int_to_bin(X) -> int_to_bin_pos(X, []). diff --git a/lib/debugger/Makefile b/lib/debugger/Makefile index 8c8b617831..3a06d72c5d 100644 --- a/lib/debugger/Makefile +++ b/lib/debugger/Makefile @@ -34,3 +34,7 @@ SPECIAL_TARGETS = # Default Subdir Targets # ---------------------------------------------------- include $(ERL_TOP)/make/otp_subdir.mk + +DIA_PLT_APPS=wx + +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/dialyzer/Makefile b/lib/dialyzer/Makefile index e4f681dcd9..53083f267d 100644 --- a/lib/dialyzer/Makefile +++ b/lib/dialyzer/Makefile @@ -42,3 +42,6 @@ SPECIAL_TARGETS = # include $(ERL_TOP)/make/otp_subdir.mk +DIA_PLT_APPS=compiler syntax_tools hipe wx + +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/diameter/Makefile b/lib/diameter/Makefile index a0195a0988..a25baeb929 100644 --- a/lib/diameter/Makefile +++ b/lib/diameter/Makefile @@ -27,7 +27,6 @@ SPECIAL_TARGETS = include $(ERL_TOP)/make/otp_subdir.mk -info: - @echo "APP_VSN = $(APP_VSN)" +DIA_PLT_APPS=ssl runtime_tools syntax_tools -.PHONY: info +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/edoc/Makefile b/lib/edoc/Makefile index 70bf1f3d48..a8258015b1 100644 --- a/lib/edoc/Makefile +++ b/lib/edoc/Makefile @@ -69,7 +69,7 @@ SPECIAL_TARGETS = include $(ERL_TOP)/make/otp_subdir.mk -.PHONY: info version +.PHONY: version version: @@ -92,9 +92,6 @@ edocs: -pa $(XMERL_DIR)/ebin -run edoc_run application \ "'$(APPNAME)'" '"."' '$(DOC_OPTS)' -info: - @echo $(HTML_FILES) - app_release: tar @@ -124,3 +121,7 @@ tar: $(APP_TAR_FILE) $(APP_TAR_FILE): $(APP_DIR) (cd $(APP_RELEASE_DIR); gtar zcf $(APP_TAR_FILE) $(DIR_NAME)) + +DIA_PLT_APPS=syntax_tools xmerl inets + +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/edoc/src/edoc.app.src b/lib/edoc/src/edoc.app.src index 43343e2ae8..834c0eb005 100644 --- a/lib/edoc/src/edoc.app.src +++ b/lib/edoc/src/edoc.app.src @@ -23,4 +23,4 @@ {applications, [compiler,kernel,stdlib,syntax_tools]}, {env, []}, {runtime_dependencies, ["xmerl-1.3.7","syntax_tools-1.6.14","stdlib-2.5", - "kernel-3.0","inets-5.10","erts-6.0"]}]}. + "kernel-3.0","erts-6.0"]}]}. diff --git a/lib/edoc/src/edoc.erl b/lib/edoc/src/edoc.erl index 62483602aa..0fdc818fae 100644 --- a/lib/edoc/src/edoc.erl +++ b/lib/edoc/src/edoc.erl @@ -259,10 +259,9 @@ opt_negations() -> %% </dd> %% <dt>{@type {doc_path, [string()]@}} %% </dt> -%% <dd>Specifies a list of URI:s pointing to directories that contain -%% EDoc-generated documentation. URI without a `scheme://' part are -%% taken as relative to `file://'. (Note that such paths must use -%% `/' as separator, regardless of the host operating system.) +%% <dd>Specifies a list of file system paths pointing to directories that +%% contain EDoc-generated documentation. All paths for applications +%% in the code path are automatically added. %% </dd> %% <dt>{@type {doclet, Module::atom()@}} %% </dt> diff --git a/lib/edoc/src/edoc_lib.erl b/lib/edoc/src/edoc_lib.erl index d00a283794..5959fa6f08 100644 --- a/lib/edoc/src/edoc_lib.erl +++ b/lib/edoc/src/edoc_lib.erl @@ -32,12 +32,12 @@ -export([count/2, lines/1, split_at/2, split_at_stop/1, split_at_space/1, filename/1, transpose/1, segment/2, get_first_sentence/1, is_space/1, strip_space/1, parse_expr/2, - parse_contact/2, escape_uri/1, join_uri/2, is_relative_uri/1, + parse_contact/2, escape_uri/1, join_uri/2, is_name/1, to_label/1, find_doc_dirs/0, find_sources/2, find_file/2, try_subdir/2, unique/1, write_file/3, write_file/4, write_info_file/3, read_info_file/1, get_doc_env/1, get_doc_env/3, copy_file/2, - uri_get/1, run_doclet/2, run_layout/2, + run_doclet/2, run_layout/2, simplify_path/1, timestr/1, datestr/1, read_encoding/2]). -import(edoc_report, [report/2, warning/2]). @@ -438,128 +438,6 @@ join_uri("", Path) -> join_uri(Base, Path) -> Base ++ "/" ++ Path. -%% Check for relative URI; "network paths" ("//...") not included! - -%% @private -is_relative_uri([$: | _]) -> - false; -is_relative_uri([$/, $/ | _]) -> - false; -is_relative_uri([$/ | _]) -> - true; -is_relative_uri([$? | _]) -> - true; -is_relative_uri([$# | _]) -> - true; -is_relative_uri([_ | Cs]) -> - is_relative_uri(Cs); -is_relative_uri([]) -> - true. - -%% @private -uri_get("file:///" ++ Path) -> - uri_get_file(Path); -uri_get("file://localhost/" ++ Path) -> - uri_get_file(Path); -uri_get("file://" ++ Path) -> - Msg = io_lib:format("cannot handle 'file:' scheme with " - "nonlocal network-path: 'file://~ts'.", - [Path]), - {error, Msg}; -uri_get("file:/" ++ Path) -> - uri_get_file(Path); -uri_get("file:" ++ Path) -> - Msg = io_lib:format("ignoring malformed URI: 'file:~ts'.", [Path]), - {error, Msg}; -uri_get("http:" ++ Path) -> - uri_get_http("http:" ++ Path); -uri_get("ftp:" ++ Path) -> - uri_get_ftp("ftp:" ++ Path); -uri_get("//" ++ Path) -> - Msg = io_lib:format("cannot access network-path: '//~ts'.", [Path]), - {error, Msg}; -uri_get([C, $:, $/ | _]=Path) when C >= $A, C =< $Z; C >= $a, C =< $z -> - uri_get_file(Path); % special case for Windows -uri_get([C, $:, $\ | _]=Path) when C >= $A, C =< $Z; C >= $a, C =< $z -> - uri_get_file(Path); % special case for Windows -uri_get(URI) -> - case is_relative_uri(URI) of - true -> - uri_get_file(URI); - false -> - Msg = io_lib:format("cannot handle URI: '~ts'.", [URI]), - {error, Msg} - end. - -uri_get_file(File0) -> - File = filename:join(?FILE_BASE, File0), - case read_file(File) of - {ok, Text} -> - {ok, Text}; - {error, R} -> - {error, file:format_error(R)} - end. - -uri_get_http(URI) -> - %% Try using option full_result=false - case catch {ok, httpc:request(get, {URI,[]}, [], - [{full_result, false}])} of - {'EXIT', _} -> - uri_get_http_r10(URI); - Result -> - uri_get_http_1(Result, URI) - end. - -uri_get_http_r10(URI) -> - %% Try most general form of request - Result = (catch {ok, httpc:request(get, {URI,[]}, [], [])}), - uri_get_http_1(Result, URI). - -uri_get_http_1(Result, URI) -> - case Result of - {ok, {ok, {200, Text}}} when is_list(Text) -> - %% new short result format - {ok, Text}; - {ok, {ok, {Status, Text}}} when is_integer(Status), is_list(Text) -> - %% new short result format when status /= 200 - Phrase = httpd_util:reason_phrase(Status), - {error, http_errmsg(Phrase, URI)}; - {ok, {ok, {{_Vsn, 200, _Phrase}, _Hdrs, Text}}} when is_list(Text) -> - %% new long result format - {ok, Text}; - {ok, {ok, {{_Vsn, _Status, Phrase}, _Hdrs, Text}}} when is_list(Text) -> - %% new long result format when status /= 200 - {error, http_errmsg(Phrase, URI)}; - {ok, {200,_Hdrs,Text}} when is_list(Text) -> - %% old result format - {ok, Text}; - {ok, {Status,_Hdrs,Text}} when is_list(Text) -> - %% old result format when status /= 200 - Phrase = httpd_util:reason_phrase(Status), - {error, http_errmsg(Phrase, URI)}; - {ok, {error, R}} -> - Reason = inet:format_error(R), - {error, http_errmsg(Reason, URI)}; - {ok, R} -> - Reason = io_lib:format("bad return value ~tP", [R, 5]), - {error, http_errmsg(Reason, URI)}; - {'EXIT', R} -> - Reason = io_lib:format("crashed with reason ~tw", [R]), - {error, http_errmsg(Reason, URI)}; - R -> - Reason = io_lib:format("uncaught throw: ~tw", [R]), - {error, http_errmsg(Reason, URI)} - end. - -http_errmsg(Reason, URI) -> - io_lib:format("http error: ~ts: '~ts'", [Reason, URI]). - -%% TODO: implement ftp access method - -uri_get_ftp(URI) -> - Msg = io_lib:format("cannot access ftp scheme yet: '~ts'.", [URI]), - {error, Msg}. - %% @private to_label([$\s | Cs]) -> to_label(Cs); @@ -754,18 +632,6 @@ read_info_file(Dir) -> {?NO_APP, []} end. -%% URI access - -uri_get_info_file(Base) -> - URI = join_uri(Base, ?INFO_FILE), - case uri_get(URI) of - {ok, Text} -> - parse_info_file(Text, URI); - {error, Msg} -> - warning("could not read '~ts': ~ts.", [URI, Msg]), - {?NO_APP, []} - end. - parse_info_file(Text, Name) -> case parse_terms(Text) of {ok, Vs} -> @@ -897,7 +763,7 @@ find_doc_dirs([]) -> get_doc_links(App, Modules, Opts) -> Path = proplists:append_values(doc_path, Opts) ++ find_doc_dirs(), - Ds = [{P, uri_get_info_file(P)} || P <- Path], + Ds = [{P, read_info_file(P)} || P <- Path], Ds1 = [{"", {App, Modules}} | Ds], D = dict:new(), make_links(Ds1, D, D). diff --git a/lib/eldap/Makefile b/lib/eldap/Makefile index 28f995e068..30a8f778ab 100644 --- a/lib/eldap/Makefile +++ b/lib/eldap/Makefile @@ -38,3 +38,6 @@ SPECIAL_TARGETS = # ---------------------------------------------------- include $(ERL_TOP)/make/otp_subdir.mk +DIA_PLT_APPS=asn ssl + +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/eldap/doc/src/eldap.xml b/lib/eldap/doc/src/eldap.xml index 790a2f4e26..4a8c2d1647 100644 --- a/lib/eldap/doc/src/eldap.xml +++ b/lib/eldap/doc/src/eldap.xml @@ -317,7 +317,7 @@ </type> <desc> <p> Modify the DN of an entry. <c>DeleteOldRDN</c> indicates - whether the current RDN should be removed from the attribute list after the after operation. + whether the current RDN should be removed from the attribute list after the operation. <c>NewSupDN</c> is the new parent that the RDN shall be moved to. If the old parent should remain as parent, <c>NewSupDN</c> shall be "".</p> <code> diff --git a/lib/erl_docgen/Makefile b/lib/erl_docgen/Makefile index 30ff2bf16e..7e9cc824ec 100644 --- a/lib/erl_docgen/Makefile +++ b/lib/erl_docgen/Makefile @@ -36,4 +36,6 @@ SPECIAL_TARGETS = # include $(ERL_TOP)/make/otp_subdir.mk +DIA_PLT_APPS=edoc xmerl +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/erl_docgen/priv/bin/specs_gen.escript b/lib/erl_docgen/priv/bin/specs_gen.escript index 859f3c21f5..116240530d 100644 --- a/lib/erl_docgen/priv/bin/specs_gen.escript +++ b/lib/erl_docgen/priv/bin/specs_gen.escript @@ -131,7 +131,7 @@ write_text(Text, File, Dir) -> ok; {error, R} -> R1 = file:format_error(R), - io:format("could not write file '~s': ~s\n", [File, R1]), + io:format("could not write file '~s': ~s\n", [OutFile, R1]), halt(2) end. diff --git a/lib/erl_docgen/priv/dtd/common.dtd b/lib/erl_docgen/priv/dtd/common.dtd index 0ccd52068b..0feb09eac2 100644 --- a/lib/erl_docgen/priv/dtd/common.dtd +++ b/lib/erl_docgen/priv/dtd/common.dtd @@ -67,7 +67,7 @@ <!ELEMENT list (item+) > <!ATTLIST list type (ordered|bulleted) "bulleted" > -<!ELEMENT taglist (tag,item+)+ > +<!ELEMENT taglist (marker*,tag,item+)+ > <!ELEMENT tag (#PCDATA|c|i|em|br|seealso|url|marker|anno)* > <!ELEMENT item (%inline;|%block;|warning|note|dont|do|quote)* > diff --git a/lib/erl_interface/Makefile b/lib/erl_interface/Makefile index 9471b0df18..633e705b3f 100644 --- a/lib/erl_interface/Makefile +++ b/lib/erl_interface/Makefile @@ -31,3 +31,5 @@ SPECIAL_TARGETS = # Default Subdir Targets # ---------------------------------------------------- include $(ERL_TOP)/make/otp_subdir.mk + +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/erl_interface/include/ei.h b/lib/erl_interface/include/ei.h index b138118f04..7d39043bb2 100644 --- a/lib/erl_interface/include/ei.h +++ b/lib/erl_interface/include/ei.h @@ -323,13 +323,24 @@ typedef struct { #define EI_SCLBK_FLG_FULL_IMPL (1 << 0) +/* + * HACK: AIX defines many socket functions like accept to be naccept, which + * pollutes the global namespace. Set up an ugly ifdef for consumers of this + * API here so they get a mangled name for AIX and the sane name elsewhere. + */ +#ifdef _AIX +#define EI_ACCEPT_NAME accept_ei +#else +#define EI_ACCEPT_NAME accept +#endif + typedef struct { int flags; int (*socket)(void **ctx, void *setup_ctx); int (*close)(void *ctx); int (*listen)(void *ctx, void *addr, int *len, int backlog); - int (*accept)(void **ctx, void *addr, int *len, unsigned tmo); + int (*EI_ACCEPT_NAME)(void **ctx, void *addr, int *len, unsigned tmo); int (*connect)(void *ctx, void *addr, int len, unsigned tmo); int (*writev)(void *ctx, const void *iov, int iovcnt, ssize_t *len, unsigned tmo); int (*write)(void *ctx, const char *buf, ssize_t *len, unsigned tmo); diff --git a/lib/erl_interface/src/connect/ei_connect.c b/lib/erl_interface/src/connect/ei_connect.c index 1b1479d2e9..f600758385 100644 --- a/lib/erl_interface/src/connect/ei_connect.c +++ b/lib/erl_interface/src/connect/ei_connect.c @@ -659,7 +659,7 @@ int ei_connect_xinit_ussi(ei_cnode* ec, const char *thishostname, return ERL_ERROR; } - ec->creation = creation & 0x3; /* 2 bits */ + ec->creation = creation; if (cookie) { if (strlen(cookie) >= sizeof(ec->ei_connect_cookie)) { @@ -698,7 +698,7 @@ int ei_connect_xinit_ussi(ei_cnode* ec, const char *thishostname, strcpy(ec->self.node,thisnodename); ec->self.num = 0; ec->self.serial = 0; - ec->self.creation = creation & 0x3; /* 2 bits */ + ec->self.creation = creation; ec->cbs = cbs; ec->setup_context = setup_context; diff --git a/lib/erl_interface/src/connect/ei_resolve.c b/lib/erl_interface/src/connect/ei_resolve.c index 225fddc784..5a8ca0c567 100644 --- a/lib/erl_interface/src/connect/ei_resolve.c +++ b/lib/erl_interface/src/connect/ei_resolve.c @@ -55,6 +55,16 @@ #include "ei_resolve.h" #include "ei_locking.h" +/* AIX has a totally different signature (allegedly shared with some other + * Unices) that isn't compatible. It turns out that the _r version isn't + * thread-safe according to curl - but bizarrely, since AIX 4.3, libc + * is thread-safe in a manner that makes the normal gethostbyname OK + * for re-entrant use. + */ +#ifdef _AIX +#undef HAVE_GETHOSTBYNAME_R +#endif + #ifdef HAVE_GETHOSTBYNAME_R int ei_init_resolve(void) @@ -75,7 +85,7 @@ int ei_init_resolve(void) static ei_mutex_t *ei_gethost_sem = NULL; #endif /* _REENTRANT */ static int ei_resolve_initialized = 0; -#ifndef __WIN32__ +#if !defined(__WIN32__) && !defined(_AIX) int h_errno; #endif diff --git a/lib/erl_interface/src/encode/encode_pid.c b/lib/erl_interface/src/encode/encode_pid.c index d14746b40f..0dfdb16372 100644 --- a/lib/erl_interface/src/encode/encode_pid.c +++ b/lib/erl_interface/src/encode/encode_pid.c @@ -25,7 +25,6 @@ int ei_encode_pid(char *buf, int *index, const erlang_pid *p) { char* s = buf + *index; - const char tag = (p->creation > 3) ? ERL_NEW_PID_EXT : ERL_PID_EXT; ++(*index); /* skip ERL_PID_EXT */ if (ei_encode_atom_len_as(buf, index, p->node, strlen(p->node), @@ -33,21 +32,17 @@ int ei_encode_pid(char *buf, int *index, const erlang_pid *p) return -1; if (buf) { - put8(s, tag); + put8(s, ERL_NEW_PID_EXT); s = buf + *index; /* now the integers */ put32be(s,p->num & 0x7fff); /* 15 bits */ put32be(s,p->serial & 0x1fff); /* 13 bits */ - if (tag == ERL_PID_EXT) { - put8(s,(p->creation & 0x03)); /* 2 bits */ - } else { - put32be(s, p->creation); /* 32 bits */ - } + put32be(s, p->creation); /* 32 bits */ } - *index += 4 + 4 + (tag == ERL_PID_EXT ? 1 : 4); + *index += 4 + 4 + 4; return 0; } diff --git a/lib/erl_interface/src/encode/encode_port.c b/lib/erl_interface/src/encode/encode_port.c index eb464380c0..0fb4018db1 100644 --- a/lib/erl_interface/src/encode/encode_port.c +++ b/lib/erl_interface/src/encode/encode_port.c @@ -25,7 +25,6 @@ int ei_encode_port(char *buf, int *index, const erlang_port *p) { char *s = buf + *index; - const char tag = p->creation > 3 ? ERL_NEW_PORT_EXT : ERL_PORT_EXT; ++(*index); /* skip ERL_PORT_EXT */ if (ei_encode_atom_len_as(buf, index, p->node, strlen(p->node), ERLANG_UTF8, @@ -33,19 +32,15 @@ int ei_encode_port(char *buf, int *index, const erlang_port *p) return -1; } if (buf) { - put8(s, tag); + put8(s, ERL_NEW_PORT_EXT); s = buf + *index; /* now the integers */ put32be(s,p->id & 0x0fffffff /* 28 bits */); - if (tag == ERL_PORT_EXT) { - put8(s,(p->creation & 0x03)); - } else { - put32be(s, p->creation); - } + put32be(s, p->creation); } - *index += 4 + (tag == ERL_PORT_EXT ? 1 : 4); + *index += 4 + 4; return 0; } diff --git a/lib/erl_interface/src/encode/encode_ref.c b/lib/erl_interface/src/encode/encode_ref.c index 5ccfc32c6d..8c2e0a25f7 100644 --- a/lib/erl_interface/src/encode/encode_ref.c +++ b/lib/erl_interface/src/encode/encode_ref.c @@ -24,7 +24,6 @@ int ei_encode_ref(char *buf, int *index, const erlang_ref *p) { - const char tag = (p->creation > 3) ? ERL_NEWER_REFERENCE_EXT : ERL_NEW_REFERENCE_EXT; char *s = buf + *index; int i; @@ -37,7 +36,7 @@ int ei_encode_ref(char *buf, int *index, const erlang_ref *p) /* Always encode as an extended reference; all participating parties are now expected to be able to decode extended references. */ if (buf) { - put8(s, tag); + put8(s, ERL_NEWER_REFERENCE_EXT); /* first, number of integers */ put16be(s, p->len); @@ -46,15 +45,12 @@ int ei_encode_ref(char *buf, int *index, const erlang_ref *p) s = buf + *index; /* now the integers */ - if (tag == ERL_NEW_REFERENCE_EXT) - put8(s,(p->creation & 0x03)); - else - put32be(s, p->creation); + put32be(s, p->creation); for (i = 0; i < p->len; i++) put32be(s,p->n[i]); } - *index += p->len*4 + (tag == ERL_NEW_REFERENCE_EXT ? 1 : 4); + *index += p->len*4 + 4; return 0; } diff --git a/lib/erl_interface/src/epmd/ei_epmd.h b/lib/erl_interface/src/epmd/ei_epmd.h index ac153b6e66..597a955676 100644 --- a/lib/erl_interface/src/epmd/ei_epmd.h +++ b/lib/erl_interface/src/epmd/ei_epmd.h @@ -25,8 +25,8 @@ #endif #ifndef EI_DIST_HIGH -#define EI_DIST_HIGH 5 /* R4 and later */ -#define EI_DIST_LOW 1 /* R3 and earlier */ +#define EI_DIST_HIGH 6 /* OTP 23 and later */ +#define EI_DIST_LOW 5 /* OTP R4 - 22 */ #endif #ifndef EPMD_PORT @@ -45,6 +45,7 @@ #ifndef EI_EPMD_ALIVE2_REQ #define EI_EPMD_ALIVE2_REQ 120 #define EI_EPMD_ALIVE2_RESP 121 +#define EI_EPMD_ALIVE2_X_RESP 118 #define EI_EPMD_PORT2_REQ 122 #define EI_EPMD_PORT2_RESP 119 #define EI_EPMD_STOP_REQ 's' diff --git a/lib/erl_interface/src/epmd/epmd_publish.c b/lib/erl_interface/src/epmd/epmd_publish.c index 20b8e867e8..ef8a5d6b70 100644 --- a/lib/erl_interface/src/epmd/epmd_publish.c +++ b/lib/erl_interface/src/epmd/epmd_publish.c @@ -68,7 +68,8 @@ static int ei_epmd_r4_publish (int port, const char *alive, unsigned ms) int nlen = strlen(alive); int len = elen + nlen + 13; /* hard coded: be careful! */ int n; - int err, res, creation; + int err, response, res; + unsigned creation; ssize_t dlen; unsigned tmo = ms == 0 ? EI_SCLBK_INF_TMO : ms; @@ -124,8 +125,10 @@ static int ei_epmd_r4_publish (int port, const char *alive, unsigned ms) /* Don't close fd here! It keeps us registered with epmd */ s = buf; - if (((res=get8(s)) != EI_EPMD_ALIVE2_RESP)) { /* response */ - EI_TRACE_ERR1("ei_epmd_r4_publish","<- unknown (%d)",res); + response = get8(s); + if (response != EI_EPMD_ALIVE2_RESP && + response != EI_EPMD_ALIVE2_X_RESP) { + EI_TRACE_ERR1("ei_epmd_r4_publish","<- unknown (%d)",response); EI_TRACE_ERR0("ei_epmd_r4_publish","-> CLOSE"); ei_close__(fd); erl_errno = EIO; @@ -141,18 +144,21 @@ static int ei_epmd_r4_publish (int port, const char *alive, unsigned ms) return -1; } - creation = get16be(s); + if (response == EI_EPMD_ALIVE2_RESP) + creation = get16be(s); + else /* EI_EPMD_ALIVE2_X_RESP */ + creation = get32be(s); EI_TRACE_CONN2("ei_epmd_r4_publish", - " result=%d (ok) creation=%d",res,creation); + " result=%d (ok) creation=%u",res,creation); - /* probably should save fd so we can close it later... */ - /* epmd_saveconn(OPEN,fd,alive); */ + /* + * Would be nice to somehow use the nice "unique" creation value + * received here from epmd instead of using the crappy one + * passed (already) to ei_connect_init. + */ - /* return the creation number, for no good reason */ - /* return creation;*/ - - /* no - return the descriptor */ + /* return the descriptor */ return fd; } diff --git a/lib/erl_interface/src/misc/ei_portio.c b/lib/erl_interface/src/misc/ei_portio.c index bccc86c1b1..bfe67a732c 100644 --- a/lib/erl_interface/src/misc/ei_portio.c +++ b/lib/erl_interface/src/misc/ei_portio.c @@ -622,7 +622,7 @@ int ei_accept_ctx_t__(ei_socket_callbacks *cbs, void **ctx, } while (error == EINTR); } do { - error = cbs->accept(ctx, addr, len, ms); + error = cbs->EI_ACCEPT_NAME(ctx, addr, len, ms); } while (error == EINTR); return error; } diff --git a/lib/erl_interface/src/prog/erl_call.c b/lib/erl_interface/src/prog/erl_call.c index ab91157035..dce2ecdec2 100644 --- a/lib/erl_interface/src/prog/erl_call.c +++ b/lib/erl_interface/src/prog/erl_call.c @@ -292,8 +292,7 @@ int erl_call(int argc, char **argv) flags.cookie = NULL; } - /* FIXME decide how many bits etc or leave to connect_xinit? */ - creation = (time(NULL) % 3) + 1; /* "random" */ + creation = time(NULL) + 1; /* "random" */ if (flags.hidden == NULL) { /* As default we are c17@gethostname */ diff --git a/lib/erl_interface/test/erl_eterm_SUITE.erl b/lib/erl_interface/test/erl_eterm_SUITE.erl index 77910a9fc7..4605293c74 100644 --- a/lib/erl_interface/test/erl_eterm_SUITE.erl +++ b/lib/erl_interface/test/erl_eterm_SUITE.erl @@ -73,6 +73,10 @@ -import(runner, [get_term/1]). +-define(REFERENCE_EXT, $e). +-define(NEW_REFERENCE_EXT, $r). +-define(NEWER_REFERENCE_EXT, $Z). + %% This test suite controls the running of the C language functions %% in eterm_test.c and print_term.c. @@ -1026,9 +1030,11 @@ cnode_1(Config) when is_list(Config) -> check_ref(Ref) -> case bin_ext_type(Ref) of - 101 -> + ?REFERENCE_EXT -> ct:fail(oldref); - 114 -> + ?NEW_REFERENCE_EXT -> + ok; + ?NEWER_REFERENCE_EXT -> ok; Type -> ct:fail({type, Type}) diff --git a/lib/et/Makefile b/lib/et/Makefile index f0bb7be211..1b89cff83e 100644 --- a/lib/et/Makefile +++ b/lib/et/Makefile @@ -35,3 +35,6 @@ SPECIAL_TARGETS = # ---------------------------------------------------- include $(ERL_TOP)/make/otp_subdir.mk +DIA_PLT_APPS=runtime_tools wx + +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/eunit/Makefile b/lib/eunit/Makefile index 15dae19896..d0dd447a6c 100644 --- a/lib/eunit/Makefile +++ b/lib/eunit/Makefile @@ -48,13 +48,7 @@ SPECIAL_TARGETS = # include $(ERL_TOP)/make/otp_subdir.mk - -.PHONY: info version - -info: - @echo "APP_RELEASE_DIR: $(APP_RELEASE_DIR)" - @echo "APP_DIR: $(APP_DIR)" - @echo "APP_TAR_FILE: $(APP_TAR_FILE)" +.PHONY: version version: @echo "$(VSN)" @@ -94,3 +88,5 @@ tar: $(APP_TAR_FILE) $(APP_TAR_FILE): $(APP_DIR) (cd $(APP_RELEASE_DIR); gtar zcf $(APP_TAR_FILE) $(DIR_NAME)) + +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/eunit/src/eunit_surefire.erl b/lib/eunit/src/eunit_surefire.erl index 2b9f82b075..002a069a92 100644 --- a/lib/eunit/src/eunit_surefire.erl +++ b/lib/eunit/src/eunit_surefire.erl @@ -451,6 +451,11 @@ escape_xml([$< | Tail], Acc, ForAttr) -> escape_xml(Tail, [$;, $t, $l, $& | Acc] escape_xml([$> | Tail], Acc, ForAttr) -> escape_xml(Tail, [$;, $t, $g, $& | Acc], ForAttr); escape_xml([$& | Tail], Acc, ForAttr) -> escape_xml(Tail, [$;, $p, $m, $a, $& | Acc], ForAttr); escape_xml([$" | Tail], Acc, true) -> escape_xml(Tail, [$;, $t, $o, $u, $q, $& | Acc], true); % " +escape_xml([Char | Tail], Acc, ForAttr) when + Char == $\n; Char == $\r; Char == $\t -> escape_xml(Tail, [Char | Acc], ForAttr); +%% Strip C0 control codes which are not allowed in XML 1.0 +escape_xml([Char | Tail], Acc, ForAttr) when + 0 =< Char, Char =< 31 -> escape_xml(Tail, Acc, ForAttr); escape_xml([Char | Tail], Acc, ForAttr) when is_integer(Char) -> escape_xml(Tail, [Char | Acc], ForAttr). %% the input may be utf8 or latin1; the resulting list is unicode diff --git a/lib/eunit/test/Makefile b/lib/eunit/test/Makefile index b6ece61b43..7c1e56c867 100644 --- a/lib/eunit/test/Makefile +++ b/lib/eunit/test/Makefile @@ -22,6 +22,7 @@ include $(ERL_TOP)/make/$(TARGET)/otp.mk MODULES = \ eunit_SUITE \ + tc0 \ tlatin \ tutf8 diff --git a/lib/eunit/test/eunit_SUITE.erl b/lib/eunit/test/eunit_SUITE.erl index 63bdc6c334..b9f4ea4557 100644 --- a/lib/eunit/test/eunit_SUITE.erl +++ b/lib/eunit/test/eunit_SUITE.erl @@ -21,14 +21,16 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, - app_test/1,appup_test/1,eunit_test/1,surefire_utf8_test/1,surefire_latin_test/1]). + app_test/1,appup_test/1,eunit_test/1,surefire_utf8_test/1,surefire_latin_test/1, + surefire_c0_test/1]). -include_lib("common_test/include/ct.hrl"). suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [app_test, appup_test, eunit_test, surefire_utf8_test, surefire_latin_test]. + [app_test, appup_test, eunit_test, surefire_utf8_test, surefire_latin_test, + surefire_c0_test]. groups() -> []. @@ -65,11 +67,24 @@ surefire_utf8_test(Config) when is_list(Config) -> check_surefire(tutf8), ok. +surefire_c0_test(Config) when is_list(Config) -> + ok = file:set_cwd(proplists:get_value(priv_dir, Config, ".")), + Chars = check_surefire(tc0), + %% Check that these characters were not stripped + true = lists:member($\n, Chars), + true = lists:member($\r, Chars), + true = lists:member($\t, Chars), + ok. + check_surefire(Module) -> File = "TEST-"++atom_to_list(Module)++".xml", file:delete(File), % ignore test result, some fail on purpose eunit:test(Module, [{report,{eunit_surefire,[{dir,"."}]}}]), {ok, Bin} = file:read_file(File), - [_|_] = unicode:characters_to_list(Bin, unicode), - ok.
\ No newline at end of file + Chars = unicode:characters_to_list(Bin, unicode), + %% Check that unicode decoding succeeded + [_|_] = Chars, + %% Check that file is valid XML + xmerl_scan:file(File), + Chars. diff --git a/lib/eunit/test/tc0.erl b/lib/eunit/test/tc0.erl new file mode 100644 index 0000000000..8c90633fc8 --- /dev/null +++ b/lib/eunit/test/tc0.erl @@ -0,0 +1,14 @@ +-module(tc0). + +-include_lib("eunit/include/eunit.hrl"). + +'c0_bad_output_test_'() -> + [{integer_to_list(C), fun() -> io:format("'~c'", [C]) end} + || C <- lists:seq(0, 31)]. + +'c0_bad_description_test_'() -> + [{[C], fun() -> ok end} + || C <- lists:seq(0, 31)]. + +'c0_bad_name__test'() -> + ok. diff --git a/lib/ftp/Makefile b/lib/ftp/Makefile index e0c9de42e4..a26d1de0a0 100644 --- a/lib/ftp/Makefile +++ b/lib/ftp/Makefile @@ -32,47 +32,11 @@ VSN = $(FTP_VSN) SPECIAL_TARGETS = -DIA_PLT = ./priv/plt/$(APPLICATION).plt -DIA_ANALYSIS = $(basename $(DIA_PLT)).dialyzer_analysis - - # ---------------------------------------------------- # Default Subdir Targets # ---------------------------------------------------- include $(ERL_TOP)/make/otp_subdir.mk -.PHONY: info gclean dialyzer dialyzer_plt dclean - -info: - @echo "OS: $(OS)" - @echo "DOCB: $(DOCB)" - @echo "" - @echo "FTP_VSN: $(FTP_VSN)" - @echo "APP_VSN: $(APP_VSN)" - @echo "" - @echo "DIA_PLT: $(DIA_PLT)" - @echo "DIA_ANALYSIS: $(DIA_ANALYSIS)" - @echo "" - -gclean: - git clean -fXd - -dclean: - rm -f $(DIA_PLT) - rm -f $(DIA_ANALYSIS) - -dialyzer_plt: $(DIA_PLT) - -$(DIA_PLT): - @echo "Building $(APPLICATION) plt file" - @dialyzer --build_plt \ - --output_plt $@ \ - -r ../$(APPLICATION)/ebin \ - --output $(DIA_ANALYSIS) \ - --verbose +DIA_PLT_APPS = runtime_tools ssl -dialyzer: $(DIA_PLT) - @echo "Running dialyzer on $(APPLICATION)" - @dialyzer --plt $< \ - ../$(APPLICATION)/ebin \ - --verbose +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/hipe/Makefile b/lib/hipe/Makefile index 0676484fca..4998522943 100644 --- a/lib/hipe/Makefile +++ b/lib/hipe/Makefile @@ -75,3 +75,6 @@ distclean: realclean: $(V_at)$(MAKE) MAKETARGET="realclean" all-subdirs all-subdirs-x +DIA_PLT_APPS=compiler syntax_tools + +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/hipe/icode/hipe_beam_to_icode.erl b/lib/hipe/icode/hipe_beam_to_icode.erl index 995c961e09..fce178a5e3 100644 --- a/lib/hipe/icode/hipe_beam_to_icode.erl +++ b/lib/hipe/icode/hipe_beam_to_icode.erl @@ -678,8 +678,8 @@ trans_fun([{call_fun,N}|Instructions], Env) -> Dst = [mk_var({r,0})], [hipe_icode:mk_comment('call_fun'), hipe_icode:mk_primop(Dst,call_fun,Args) | trans_fun(Instructions,Env)]; -%%--- patched_make_fun --- make_fun/make_fun2 after fixes -trans_fun([{patched_make_fun,MFA,Magic,FreeVarNum,Index}|Instructions], Env) -> +%%--- make_fun2 --- +trans_fun([{make_fun2,MFA,Index,Magic,FreeVarNum}|Instructions], Env) -> Args = extract_fun_args(FreeVarNum), Dst = [mk_var({r,0})], Fun = hipe_icode:mk_primop(Dst, @@ -1193,6 +1193,17 @@ trans_fun([{bs_get_position=Name,_,_,_}|_Instructions], _Env) -> trans_fun([{bs_set_position=Name,_,_}|_Instructions], _Env) -> nyi(Name); %%-------------------------------------------------------------------- +%% New instructions added in OTP 23. +%%-------------------------------------------------------------------- +%%--- swap --- +trans_fun([{swap,Reg1,Reg2}|Instructions], Env) -> + Var1 = mk_var(Reg1), + Var2 = mk_var(Reg2), + Temp = mk_var(new), + [hipe_icode:mk_move(Temp, Var1), + hipe_icode:mk_move(Var1, Var2), + hipe_icode:mk_move(Var2, Temp) | trans_fun(Instructions, Env)]; +%%-------------------------------------------------------------------- %%--- ERROR HANDLING --- %%-------------------------------------------------------------------- trans_fun([X|_], _) -> @@ -1935,7 +1946,7 @@ mod_find_closure_info([FunCode|Fs], CI) -> mod_find_closure_info([], CI) -> CI. -find_closure_info([{patched_make_fun,MFA={_M,_F,A},_Magic,FreeVarNum,_Index}|BeamCode], +find_closure_info([{make_fun2,{_M,_F,A}=MFA,_Index,_Magic,FreeVarNum}|BeamCode], ClosureInfo) -> NewClosure = %% A-FreeVarNum+1 (The real arity + 1 for the closure) #closure_info{mfa=MFA, arity=A-FreeVarNum+1, fv_arity=FreeVarNum}, @@ -2013,41 +2024,8 @@ split_params(N, [ArgN|OrgArgs], Args) -> %%----------------------------------------------------------------------- preprocess_code(ModuleCode) -> - PatchedCode = patch_R7_funs(ModuleCode), - ClosureInfo = find_closure_info(PatchedCode), - {PatchedCode, ClosureInfo}. - -%%----------------------------------------------------------------------- -%% Patches the "make_fun" BEAM instructions of R7 so that they also -%% contain the index that the BEAM loader generates for funs. -%% -%% The index starts from 0 and is incremented by 1 for each make_fun -%% instruction encountered. -%% -%% Retained only for compatibility with BEAM code prior to R8. -%% -%% Temporarily, it also rewrites R8-PRE-RELEASE "make_fun2" -%% instructions, since their embedded indices don't work. -%%----------------------------------------------------------------------- - -patch_R7_funs(ModuleCode) -> - patch_make_funs(ModuleCode, 0). - -patch_make_funs([FunCode0|Fs], FunIndex0) -> - {PatchedFunCode,FunIndex} = patch_make_funs(FunCode0, FunIndex0, []), - [PatchedFunCode|patch_make_funs(Fs, FunIndex)]; -patch_make_funs([], _) -> []. - -patch_make_funs([{make_fun,MFA,Magic,FreeVarNum}|Is], FunIndex, Acc) -> - Patched = {patched_make_fun,MFA,Magic,FreeVarNum,FunIndex}, - patch_make_funs(Is, FunIndex+1, [Patched|Acc]); -patch_make_funs([{make_fun2,MFA,_BogusIndex,Magic,FreeVarNum}|Is], FunIndex, Acc) -> - Patched = {patched_make_fun,MFA,Magic,FreeVarNum,FunIndex}, - patch_make_funs(Is, FunIndex+1, [Patched|Acc]); -patch_make_funs([I|Is], FunIndex, Acc) -> - patch_make_funs(Is, FunIndex, [I|Acc]); -patch_make_funs([], FunIndex, Acc) -> - {lists:reverse(Acc),FunIndex}. + ClosureInfo = find_closure_info(ModuleCode), + {ModuleCode, ClosureInfo}. %%----------------------------------------------------------------------- diff --git a/lib/inets/Makefile b/lib/inets/Makefile index 872df9d055..a7723dc0d8 100644 --- a/lib/inets/Makefile +++ b/lib/inets/Makefile @@ -32,47 +32,11 @@ VSN = $(INETS_VSN) SPECIAL_TARGETS = -DIA_PLT = ./priv/plt/$(APPLICATION).plt -DIA_ANALYSIS = $(basename $(DIA_PLT)).dialyzer_analysis - - # ---------------------------------------------------- # Default Subdir Targets # ---------------------------------------------------- include $(ERL_TOP)/make/otp_subdir.mk -.PHONY: info gclean dialyzer dialyzer_plt dclean - -info: - @echo "OS: $(OS)" - @echo "DOCB: $(DOCB)" - @echo "" - @echo "INETS_VSN: $(INETS_VSN)" - @echo "APP_VSN: $(APP_VSN)" - @echo "" - @echo "DIA_PLT: $(DIA_PLT)" - @echo "DIA_ANALYSIS: $(DIA_ANALYSIS)" - @echo "" - -gclean: - git clean -fXd - -dclean: - rm -f $(DIA_PLT) - rm -f $(DIA_ANALYSIS) - -dialyzer_plt: $(DIA_PLT) - -$(DIA_PLT): - @echo "Building $(APPLICATION) plt file" - @dialyzer --build_plt \ - --output_plt $@ \ - -r ../$(APPLICATION)/ebin \ - --output $(DIA_ANALYSIS) \ - --verbose +DIA_PLT_APPS=runtime_tools ftp mnesia ssl tftp -dialyzer: $(DIA_PLT) - @echo "Running dialyzer on $(APPLICATION)" - @dialyzer --plt $< \ - ../$(APPLICATION)/ebin \ - --verbose +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/jinterface/Makefile b/lib/jinterface/Makefile index 9cf5f3e94c..dd22d743a5 100644 --- a/lib/jinterface/Makefile +++ b/lib/jinterface/Makefile @@ -39,3 +39,4 @@ SPECIAL_TARGETS = # ---------------------------------------------------- include $(ERL_TOP)/make/otp_subdir.mk +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpErlangPid.java b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpErlangPid.java index 9cbd735751..3abdf9535f 100644 --- a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpErlangPid.java +++ b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpErlangPid.java @@ -27,7 +27,6 @@ public class OtpErlangPid extends OtpErlangObject implements Comparable<Object> // don't change this! private static final long serialVersionUID = 1664394142301803659L; - private final int tag; private final String node; private final int id; private final int serial; @@ -45,7 +44,6 @@ public class OtpErlangPid extends OtpErlangObject implements Comparable<Object> public OtpErlangPid(final OtpLocalNode self) { final OtpErlangPid p = self.createPid(); - tag = p.tag; id = p.id; serial = p.serial; creation = p.creation; @@ -67,7 +65,6 @@ public class OtpErlangPid extends OtpErlangObject implements Comparable<Object> throws OtpErlangDecodeException { final OtpErlangPid p = buf.read_pid(); - tag = p.tag; node = p.node(); id = p.id(); serial = p.serial(); @@ -118,7 +115,6 @@ public class OtpErlangPid extends OtpErlangObject implements Comparable<Object> */ protected OtpErlangPid(final int tag, final String node, final int id, final int serial, final int creation) { - this.tag = tag; this.node = node; if (tag == OtpExternal.pidTag) { this.id = id & 0x7fff; // 15 bits @@ -133,7 +129,7 @@ public class OtpErlangPid extends OtpErlangObject implements Comparable<Object> } protected int tag() { - return tag; + return OtpExternal.newPidTag; } /** diff --git a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpErlangPort.java b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpErlangPort.java index 79b5d2736c..c8648d7aa3 100644 --- a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpErlangPort.java +++ b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpErlangPort.java @@ -26,7 +26,6 @@ public class OtpErlangPort extends OtpErlangObject { // don't change this! private static final long serialVersionUID = 4037115468007644704L; - private final int tag; private final String node; private final int id; private final int creation; @@ -43,7 +42,6 @@ public class OtpErlangPort extends OtpErlangObject { private OtpErlangPort(final OtpSelf self) { final OtpErlangPort p = self.createPort(); - tag = p.tag; id = p.id; creation = p.creation; node = p.node; @@ -64,7 +62,6 @@ public class OtpErlangPort extends OtpErlangObject { throws OtpErlangDecodeException { final OtpErlangPort p = buf.read_port(); - tag = p.tag; node = p.node(); id = p.id(); creation = p.creation(); @@ -105,7 +102,6 @@ public class OtpErlangPort extends OtpErlangObject { */ public OtpErlangPort(final int tag, final String node, final int id, final int creation) { - this.tag = tag; this.node = node; if (tag == OtpExternal.portTag) { this.id = id & 0xfffffff; // 28 bits @@ -118,7 +114,7 @@ public class OtpErlangPort extends OtpErlangObject { } protected int tag() { - return tag; + return OtpExternal.newPortTag; } /** diff --git a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpErlangRef.java b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpErlangRef.java index 2165397013..2bf8d9a56b 100644 --- a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpErlangRef.java +++ b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpErlangRef.java @@ -28,7 +28,6 @@ public class OtpErlangRef extends OtpErlangObject { // don't change this! private static final long serialVersionUID = -7022666480768586521L; - private final int tag; private final String node; private final int creation; @@ -49,7 +48,6 @@ public class OtpErlangRef extends OtpErlangObject { public OtpErlangRef(final OtpLocalNode self) { final OtpErlangRef r = self.createRef(); - tag = r.tag; ids = r.ids; creation = r.creation; node = r.node; @@ -70,7 +68,6 @@ public class OtpErlangRef extends OtpErlangObject { throws OtpErlangDecodeException { final OtpErlangRef r = buf.read_ref(); - tag = r.tag; node = r.node(); creation = r.creation(); @@ -90,7 +87,6 @@ public class OtpErlangRef extends OtpErlangObject { * another arbitrary number. */ public OtpErlangRef(final String node, final int id, final int creation) { - this.tag = OtpExternal.newRefTag; this.node = node; ids = new int[1]; ids[0] = id & 0x3ffff; // 18 bits @@ -138,7 +134,6 @@ public class OtpErlangRef extends OtpErlangObject { */ public OtpErlangRef(final int tag, final String node, final int[] ids, final int creation) { - this.tag = tag; this.node = node; // use at most 3 words @@ -162,7 +157,7 @@ public class OtpErlangRef extends OtpErlangObject { } protected int tag() { - return tag; + return OtpExternal.newerRefTag; } /** diff --git a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java index 5e777c1164..917e5baf3a 100644 --- a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java +++ b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpOutputStream.java @@ -713,7 +713,7 @@ public class OtpOutputStream extends ByteArrayOutputStream { */ public void write_pid(final String node, final int id, final int serial, final int creation) { - write1(OtpExternal.pidTag); + write1(OtpExternal.newPidTag); write_atom(node); write4BE(id & 0x7fff); // 15 bits write4BE(serial & 0x1fff); // 13 bits @@ -727,20 +727,11 @@ public class OtpOutputStream extends ByteArrayOutputStream { * the pid */ public void write_pid(OtpErlangPid pid) { - write1(pid.tag()); + write1(OtpExternal.newPidTag); write_atom(pid.node()); write4BE(pid.id()); write4BE(pid.serial()); - switch (pid.tag()) { - case OtpExternal.pidTag: - write1(pid.creation()); - break; - case OtpExternal.newPidTag: - write4BE(pid.creation()); - break; - default: - throw new AssertionError("Invalid pid tag " + pid.tag()); - } + write4BE(pid.creation()); } @@ -758,7 +749,7 @@ public class OtpOutputStream extends ByteArrayOutputStream { * be used. */ public void write_port(final String node, final int id, final int creation) { - write1(OtpExternal.portTag); + write1(OtpExternal.newPortTag); write_atom(node); write4BE(id & 0xfffffff); // 28 bits write1(creation & 0x3); // 2 bits @@ -771,19 +762,10 @@ public class OtpOutputStream extends ByteArrayOutputStream { * the port. */ public void write_port(OtpErlangPort port) { - write1(port.tag()); + write1(OtpExternal.newPortTag); write_atom(port.node()); write4BE(port.id()); - switch (port.tag()) { - case OtpExternal.portTag: - write1(port.creation()); - break; - case OtpExternal.newPortTag: - write4BE(port.creation()); - break; - default: - throw new AssertionError("Invalid port tag " + port.tag()); - } + write4BE(port.creation()); } /** @@ -829,7 +811,7 @@ public class OtpOutputStream extends ByteArrayOutputStream { arity = 3; // max 3 words in ref } - write1(OtpExternal.newRefTag); + write1(OtpExternal.newerRefTag); // how many id values write2BE(arity); @@ -857,24 +839,12 @@ public class OtpOutputStream extends ByteArrayOutputStream { int[] ids = ref.ids(); int arity = ids.length; - write1(ref.tag()); + write1(OtpExternal.newerRefTag); write2BE(arity); write_atom(ref.node()); + write4BE(ref.creation()); - switch (ref.tag()) { - case OtpExternal.newRefTag: - write1(ref.creation()); - write4BE(ids[0] & 0x3ffff); // first word gets truncated to 18 bits - break; - case OtpExternal.newerRefTag: - write4BE(ref.creation()); - write4BE(ids[0]); // full first word - break; - default: - throw new AssertionError("Invalid ref tag " + ref.tag()); - } - - for (int i = 1; i < arity; i++) { + for (int i = 0; i < arity; i++) { write4BE(ids[i]); } } diff --git a/lib/kernel/Makefile b/lib/kernel/Makefile index b956f5eaf5..5ab8ac63b9 100644 --- a/lib/kernel/Makefile +++ b/lib/kernel/Makefile @@ -34,3 +34,5 @@ SPECIAL_TARGETS = # Default Subdir Targets # ---------------------------------------------------- include $(ERL_TOP)/make/otp_subdir.mk + +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/kernel/doc/src/file.xml b/lib/kernel/doc/src/file.xml index fc25e83d40..2c09c84b25 100644 --- a/lib/kernel/doc/src/file.xml +++ b/lib/kernel/doc/src/file.xml @@ -939,6 +939,10 @@ f.txt: {person, "kalle", 25}. support for POSIX <c>O_SYNC</c> or equivalent, use of the <c>sync</c> flag causes <c>open</c> to return <c>{error, enotsup}</c>.</p> </item> + <tag><c>directory</c></tag> + <item> + <p>Allows <c>open</c> to work on directories.</p> + </item> </taglist> <p>Returns:</p> <taglist> @@ -953,10 +957,9 @@ f.txt: {person, "kalle", 25}. </item> </taglist> <p><c><anno>IoDevice</anno></c> is really the pid of the process that - handles the file. This process is linked to the process - that originally opened the file. If any process to which - the <c><anno>IoDevice</anno></c> is linked terminates, the file is - closed and the process itself is terminated. + handles the file. This process monitors the process that originally + opened the file (the owner process). If the owner process terminates, + the file is closed and the process itself terminates too. An <c><anno>IoDevice</anno></c> returned from this call can be used as an argument to the I/O functions (see <seealso marker="stdlib:io"><c>io(3)</c></seealso>).</p> @@ -985,8 +988,10 @@ f.txt: {person, "kalle", 25}. </item> <tag><c>enotdir</c></tag> <item> - <p>A component of the filename is not a directory. On some - platforms, <c>enoent</c> is returned instead.</p> + <p>A component of the filename is not a directory, or the + filename itself is not a directory if <c>directory</c> + mode was specified. On some platforms, <c>enoent</c> is + returned instead.</p> </item> <tag><c>enospc</c></tag> <item> @@ -1438,7 +1443,11 @@ f.txt: {person, "kalle", 25}. break this module's atomicity guarantees as it can race with a concurrent call to <seealso marker="#write_file_info/2"><c>write_file_info/1,2</c> - </seealso></p> + </seealso>.</p> + <p>This option has no effect when the function is + given an I/O device instead of a file name. Use + <seealso marker="#open/2"><c>open/2</c></seealso> with the + <c>raw</c> mode to obtain a file descriptor first.</p> <note> <p>As file times are stored in POSIX time on most OS, it is faster to query file information with option <c>posix</c>.</p> diff --git a/lib/kernel/doc/src/gen_udp.xml b/lib/kernel/doc/src/gen_udp.xml index 14819aa938..9ecadc5498 100644 --- a/lib/kernel/doc/src/gen_udp.xml +++ b/lib/kernel/doc/src/gen_udp.xml @@ -305,4 +305,3 @@ </func> </funcs> </erlref> - diff --git a/lib/kernel/doc/src/seq_trace.xml b/lib/kernel/doc/src/seq_trace.xml index aa29223dd0..aa9067f082 100644 --- a/lib/kernel/doc/src/seq_trace.xml +++ b/lib/kernel/doc/src/seq_trace.xml @@ -107,6 +107,12 @@ seq_trace:set_token(OldToken), % activate the trace token again enables/disables tracing on message sending. Default is <c>false</c>.</p> </item> + <tag><c>set_token('spawn', <anno>Bool</anno>)</c></tag> + <item> + <p>A trace token flag (<c>true | false</c>) which + enables/disables tracing on process spawning. Default is + <c>false</c>.</p> + </item> <tag><c>set_token('receive', <anno>Bool</anno>)</c></tag> <item> <p>A trace token flag (<c>true | false</c>) which @@ -257,7 +263,12 @@ TimeStamp = {Seconds, Milliseconds, Microseconds} <tag><c>{send, Serial, From, To, Message}</c></tag> <item> <p>Used when a process <c>From</c> with its trace token flag - <c>print</c> set to <c>true</c> has sent a message.</p> + <c>send</c> set to <c>true</c> has sent a message.</p> + </item> + <tag><c>{spawn, Serial, Parent, Child, _}</c></tag> + <item> + <p>Used when a process <c>Parent</c> with its trace token flag + <c>spawn</c> set to <c>true</c> has spawned a process.</p> </item> <tag><c>{'receive', Serial, From, To, Message}</c></tag> <item> @@ -295,8 +306,8 @@ TimeStamp = {Seconds, Milliseconds, Microseconds} is initiated by a single message. In short, it works as follows:</p> <p>Each process has a <em>trace token</em>, which can be empty or not empty. When not empty, the trace token can be seen as - the tuple <c>{Label, Flags, Serial, From}</c>. The trace token is - passed invisibly with each message.</p> + the tuple <c>{Label, Flags, Serial, From}</c>. The trace token is passed + invisibly to spawned processes and with each message sent.</p> <p>To start a sequential trace, the user must explicitly set the trace token in the process that will send the first message in a sequence.</p> @@ -306,9 +317,10 @@ TimeStamp = {Seconds, Milliseconds, Microseconds} <p>On each Erlang node, a process can be set as the <em>system tracer</em>. This process will receive trace messages each time a message with a trace token is sent or received (if the trace - token flag <c>send</c> or <c>'receive'</c> is set). The system - tracer can then print each trace event, write it to a file, or - whatever suitable.</p> + token flag <c>send</c> or <c>'receive'</c> is set), and when a process + with a non-empty trace token spawns another (if the trace token flag + <c>spawn</c> is set). The system tracer can then print each trace event, + write it to a file, or whatever suitable.</p> <note> <p>The system tracer only receives those trace events that occur locally within the Erlang node. To get the whole picture @@ -322,10 +334,9 @@ TimeStamp = {Seconds, Milliseconds, Microseconds} <section> <title>Trace Token</title> - <p>Each process has a current trace token. Initially, the token is - empty. When the process sends a message to another process, a - copy of the current token is sent "invisibly" along with - the message.</p> + <p>Each process has a current trace token, which is copied from the process + that spawned it. When a process sends a message to another process, a + copy of the current token is sent "invisibly" along with the message.</p> <p>The current token of a process is set in one of the following two ways:</p> <list type="bulleted"> @@ -354,8 +365,9 @@ TimeStamp = {Seconds, Milliseconds, Microseconds} <p>The algorithm for updating <c>Serial</c> can be described as follows:</p> <p>Let each process have two counters, <c>prev_cnt</c> and - <c>curr_cnt</c>, both are set to <c>0</c> when a process is created. - The counters are updated at the following occasions:</p> + <c>curr_cnt</c>, both are set to <c>0</c> when a process is created + outside of a trace sequence. The counters are updated at the following + occasions:</p> <list type="bulleted"> <item> <p><em>When the process is about to send a message and the trace token @@ -370,6 +382,16 @@ tcurr := curr_cnt</pre> passed along with the message.</p> </item> <item> + <p><em>When the process is about to spawn another process and the trace + token is not empty.</em></p> + <p>The counters of the parent process are updated in the same way as + for send above. The trace token is then passed to the child process, + whose counters will be set as follows:</p> + <code> +curr_cnt := tcurr +prev_cnt := tcurr</code> + </item> + <item> <p><em>When the process calls</em> <c>seq_trace:print(Label, Info)</c>, <c>Label</c> <em>matches the label part of the trace token and the trace token print flag is <c>true</c>.</em></p> @@ -487,9 +509,9 @@ tracer() -> print_trace(Label,TraceInfo,false); {seq_trace,Label,TraceInfo,Ts} -> print_trace(Label,TraceInfo,Ts); - Other -> ignore + _Other -> ignore end, - tracer(). + tracer(). print_trace(Label,TraceInfo,false) -> io:format("~p:",[Label]), @@ -504,8 +526,11 @@ print_trace({'receive',Serial,From,To,Message}) -> io:format("~p Received ~p FROM ~p WITH~n~p~n", [To,Serial,From,Message]); print_trace({send,Serial,From,To,Message}) -> - io:format("~p Sent ~p TO ~p WITH~n~p~n", - [From,Serial,To,Message]).</code> + io:format("~p Sent ~p TO ~p WITH~n~p~n", + [From,Serial,To,Message]); +print_trace({spawn,Serial,Parent,Child,_}) -> + io:format("~p Spawned ~p AT ~p~n", + [Parent,Child,Serial]).</code> <p>The code that creates a process that runs this tracer function and sets that process as the system tracer can look like this:</p> <code type="none"> diff --git a/lib/kernel/src/erl_epmd.erl b/lib/kernel/src/erl_epmd.erl index 7a14e2635c..f31a1722ce 100644 --- a/lib/kernel/src/erl_epmd.erl +++ b/lib/kernel/src/erl_epmd.erl @@ -33,10 +33,10 @@ -define(erlang_daemon_port, 4369). -endif. -ifndef(epmd_dist_high). --define(epmd_dist_high, 4370). +-define(epmd_dist_high, 6). -endif. -ifndef(epmd_dist_low). --define(epmd_dist_low, 4370). +-define(epmd_dist_low, 5). -endif. %% External exports @@ -342,6 +342,13 @@ wait_for_reg_reply(Socket, SoFar) -> receive {tcp, Socket, Data0} -> case SoFar ++ Data0 of + [$v, Result, A, B, C, D] -> + case Result of + 0 -> + {alive, Socket, ?u32(A, B, C, D)}; + _ -> + {error, duplicate_name} + end; [$y, Result, A, B] -> case Result of 0 -> diff --git a/lib/kernel/src/erts_debug.erl b/lib/kernel/src/erts_debug.erl index e6a30d0b92..7b9067d079 100644 --- a/lib/kernel/src/erts_debug.erl +++ b/lib/kernel/src/erts_debug.erl @@ -40,6 +40,15 @@ lc_graph/0, lc_graph_to_dot/2, lc_graph_merge/2, alloc_blocks_size/1]). +%% Reroutes calls to the given MFA to error_handler:breakpoint/3 +%% +%% Note that this is potentially unsafe as compiled code may assume that the +%% targeted function returns a specific type, triggering undefined behavior if +%% this function were to return something else. +%% +%% For reference, the debugger avoids the issue by purging the affected module +%% and interpreting all functions in the module, ensuring that no assumptions +%% are made with regard to return or argument types. -spec breakpoint(MFA, Flag) -> non_neg_integer() when MFA :: {Module :: module(), Function :: atom(), @@ -92,7 +101,7 @@ copy_shared(_) -> -spec get_internal_state(W) -> term() when W :: reds_left | node_and_dist_references | monitoring_nodes - | next_pid | 'DbTable_words' | check_io_debug + | next_pid | 'DbTable_words' | check_io_debug | lc_graph | process_info_args | processes | processes_bif_info | max_atom_out_cache_index | nbalance | available_internal_state | force_heap_frags | memory diff --git a/lib/kernel/src/file.erl b/lib/kernel/src/file.erl index 1d4e37196c..cde03ce1c4 100644 --- a/lib/kernel/src/file.erl +++ b/lib/kernel/src/file.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2018. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -239,20 +239,30 @@ make_dir(Name) -> del_dir(Name) -> check_and_call(del_dir, [file_name(Name)]). --spec read_file_info(Filename) -> {ok, FileInfo} | {error, Reason} when - Filename :: name_all(), +-spec read_file_info(File) -> {ok, FileInfo} | {error, Reason} when + File :: name_all() | io_device(), FileInfo :: file_info(), Reason :: posix() | badarg. +read_file_info(IoDevice) + when is_pid(IoDevice); is_record(IoDevice, file_descriptor) -> + read_file_info(IoDevice, []); + read_file_info(Name) -> check_and_call(read_file_info, [file_name(Name)]). --spec read_file_info(Filename, Opts) -> {ok, FileInfo} | {error, Reason} when - Filename :: name_all(), +-spec read_file_info(File, Opts) -> {ok, FileInfo} | {error, Reason} when + File :: name_all() | io_device(), Opts :: [file_info_option()], FileInfo :: file_info(), Reason :: posix() | badarg. +read_file_info(IoDevice, Opts) when is_pid(IoDevice), is_list(Opts) -> + file_request(IoDevice, {read_handle_info, Opts}); + +read_file_info(#file_descriptor{module = Module} = Handle, Opts) when is_list(Opts) -> + Module:read_handle_info(Handle, Opts); + read_file_info(Name, Opts) when is_list(Opts) -> Args = [file_name(Name), Opts], case check_args(Args) of @@ -460,7 +470,7 @@ raw_write_file_info(Name, #file_info{} = Info) -> -spec open(File, Modes) -> {ok, IoDevice} | {error, Reason} when File :: Filename | iodata(), Filename :: name_all(), - Modes :: [mode() | ram], + Modes :: [mode() | ram | directory], IoDevice :: io_device(), Reason :: posix() | badarg | system_limit. @@ -545,7 +555,7 @@ allocate(#file_descriptor{module = Module} = Handle, Offset, Length) -> | {no_translation, unicode, latin1}. read(File, Sz) when (is_pid(File) orelse is_atom(File)), is_integer(Sz), Sz >= 0 -> - case io:request(File, {get_chars, '', Sz}) of + case io:request(File, {get_chars, latin1, '', Sz}) of Data when is_list(Data); is_binary(Data) -> {ok, Data}; Other -> @@ -566,7 +576,7 @@ read(_, _) -> | {no_translation, unicode, latin1}. read_line(File) when (is_pid(File) orelse is_atom(File)) -> - case io:request(File, {get_line, ''}) of + case io:request(File, {get_line, latin1, ''}) of Data when is_list(Data); is_binary(Data) -> {ok, Data}; Other -> @@ -1143,7 +1153,7 @@ path_script(Path, File, Bs) -> {ok, IoDevice, FullName} | {error, Reason} when Path :: [Dir :: name_all()], Filename :: name_all(), - Modes :: [mode()], + Modes :: [mode() | directory], IoDevice :: io_device(), FullName :: filename_all(), Reason :: posix() | badarg | system_limit. diff --git a/lib/kernel/src/file_io_server.erl b/lib/kernel/src/file_io_server.erl index 34d5497a4a..c03fbb548a 100644 --- a/lib/kernel/src/file_io_server.erl +++ b/lib/kernel/src/file_io_server.erl @@ -314,6 +314,14 @@ file_request(truncate, Reply -> std_reply(Reply, State) end; +file_request({read_handle_info, Opts}, + #state{handle=Handle}=State) -> + case ?CALL_FD(Handle, read_handle_info, [Opts]) of + {error,Reason}=Reply -> + {stop,Reason,Reply,State}; + Reply -> + {reply,Reply,State} + end; file_request(Unknown, #state{}=State) -> Reason = {request, Unknown}, diff --git a/lib/kernel/src/group.erl b/lib/kernel/src/group.erl index 5625ae6eb7..8c0ced8678 100644 --- a/lib/kernel/src/group.erl +++ b/lib/kernel/src/group.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2018. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -795,12 +795,6 @@ save_line({stack, U, _L, D}, Line) -> {stack, U, Line, D}. get_lines(Ls) -> get_all_lines(Ls). -%get_lines({stack, U, {}, []}) -> -% U; -%get_lines({stack, U, {}, D}) -> -% tl(lists:reverse(D, U)); -%get_lines({stack, U, L, D}) -> -% get_lines({stack, U, {}, [L|D]}). %% There's a funny behaviour whenever the line stack doesn't have a "\n" %% at its end -- get_lines() seemed to work on the assumption it *will* be diff --git a/lib/kernel/src/group_history.erl b/lib/kernel/src/group_history.erl index 9745848992..1fab2ba14e 100644 --- a/lib/kernel/src/group_history.erl +++ b/lib/kernel/src/group_history.erl @@ -44,7 +44,7 @@ load() -> wait_for_kernel_safe_sup(), case history_status() of enabled -> - case open_log() of + try open_log() of {ok, ?LOG_NAME} -> read_full_log(?LOG_NAME); {repaired, ?LOG_NAME, {recovered, Good}, {badbytes, Bad}} -> @@ -68,6 +68,10 @@ load() -> handle_open_error(Reason), disable_history(), [] + catch + % disk_log shut down abruptly, possibly because + % the node is shutting down. Ignore it. + exit:_ -> [] end; _ -> [] @@ -127,11 +131,15 @@ repair_log(Name) -> %% Return whether the shell history is enabled or not -spec history_status() -> enabled | disabled. history_status() -> - case is_user() orelse application:get_env(kernel, shell_history) of - true -> disabled; % don't run for user proc - {ok, enabled} -> enabled; - undefined -> ?DEFAULT_STATUS; - _ -> disabled + %% Don't run for user proc or if the emulator's tearing down + Skip = is_user() orelse not init_running(), + case application:get_env(kernel, shell_history) of + {ok, enabled} when not Skip -> + enabled; + undefined when not Skip -> + ?DEFAULT_STATUS; + _ -> + disabled end. %% Return whether the user process is running this @@ -142,6 +150,14 @@ is_user() -> _ -> false end. +%% Return if the system is running (not stopping) +-spec init_running() -> boolean(). +init_running() -> + case init:get_status() of + {stopping, _} -> false; + _ -> true + end. + %% Open a disk_log file while ensuring the required path is there. open_log() -> Opts = log_options(), @@ -322,7 +338,7 @@ show_unexpected_warning({M,F,A}, Term) -> show_unexpected_close_warning() -> show('$#erlang-history-unexpected-close', - "The shell log file has mysteriousy closed. Ignoring " + "The shell log file has mysteriously closed. Ignoring " "currently unread history.~n", []). show_size_warning(_Current, _New) -> diff --git a/lib/kernel/src/raw_file_io_compressed.erl b/lib/kernel/src/raw_file_io_compressed.erl index f6ac6eaffc..1a6de3a4eb 100644 --- a/lib/kernel/src/raw_file_io_compressed.erl +++ b/lib/kernel/src/raw_file_io_compressed.erl @@ -21,7 +21,8 @@ -export([close/1, sync/1, datasync/1, truncate/1, advise/4, allocate/3, position/2, write/2, pwrite/2, pwrite/3, - read_line/1, read/2, pread/2, pread/3]). + read_line/1, read/2, pread/2, pread/3, + read_handle_info/2]). %% OTP internal. -export([ipread_s32bu_p32bu/3, sendfile/8]). @@ -118,6 +119,9 @@ ipread_s32bu_p32bu(Fd, Offset, MaxSize) -> sendfile(_,_,_,_,_,_,_,_) -> {error, enotsup}. +read_handle_info(Fd, Opts) -> + wrap_call(Fd, [Opts]). + wrap_call(Fd, Command) -> {_Owner, Pid} = get_fd_data(Fd), try gen_statem:call(Pid, Command, infinity) of diff --git a/lib/kernel/src/raw_file_io_delayed.erl b/lib/kernel/src/raw_file_io_delayed.erl index 5644717aaa..f9b89270f1 100644 --- a/lib/kernel/src/raw_file_io_delayed.erl +++ b/lib/kernel/src/raw_file_io_delayed.erl @@ -23,7 +23,8 @@ -export([close/1, sync/1, datasync/1, truncate/1, advise/4, allocate/3, position/2, write/2, pwrite/2, pwrite/3, - read_line/1, read/2, pread/2, pread/3]). + read_line/1, read/2, pread/2, pread/3, + read_handle_info/2]). %% OTP internal. -export([ipread_s32bu_p32bu/3, sendfile/8]). @@ -304,6 +305,9 @@ ipread_s32bu_p32bu(Fd, Offset, MaxSize) -> sendfile(_,_,_,_,_,_,_,_) -> {error, enotsup}. +read_handle_info(Fd, Opts) -> + wrap_call(Fd, [Opts]). + wrap_call(Fd, Command) -> #{ pid := Pid } = get_fd_data(Fd), try gen_statem:call(Pid, Command, infinity) of diff --git a/lib/kernel/src/raw_file_io_list.erl b/lib/kernel/src/raw_file_io_list.erl index 2e16e63f0e..e4fe434e13 100644 --- a/lib/kernel/src/raw_file_io_list.erl +++ b/lib/kernel/src/raw_file_io_list.erl @@ -21,7 +21,8 @@ -export([close/1, sync/1, datasync/1, truncate/1, advise/4, allocate/3, position/2, write/2, pwrite/2, pwrite/3, - read_line/1, read/2, pread/2, pread/3]). + read_line/1, read/2, pread/2, pread/3, + read_handle_info/2]). %% OTP internal. -export([ipread_s32bu_p32bu/3, sendfile/8]). @@ -126,3 +127,7 @@ sendfile(Fd, Dest, Offset, Bytes, ChunkSize, Headers, Trailers, Flags) -> Args = [Dest, Offset, Bytes, ChunkSize, Headers, Trailers, Flags], PrivateFd = Fd#file_descriptor.data, ?CALL_FD(PrivateFd, sendfile, Args). + +read_handle_info(Fd, Opts) -> + PrivateFd = Fd#file_descriptor.data, + ?CALL_FD(PrivateFd, read_handle_info, [Opts]). diff --git a/lib/kernel/src/seq_trace.erl b/lib/kernel/src/seq_trace.erl index f0bd1fabe9..bc023007bf 100644 --- a/lib/kernel/src/seq_trace.erl +++ b/lib/kernel/src/seq_trace.erl @@ -20,12 +20,14 @@ -module(seq_trace). --define(SEQ_TRACE_SEND, 1). %(1 << 0) --define(SEQ_TRACE_RECEIVE, 2). %(1 << 1) --define(SEQ_TRACE_PRINT, 4). %(1 << 2) --define(SEQ_TRACE_NOW_TIMESTAMP, 8). %(1 << 3) --define(SEQ_TRACE_STRICT_MON_TIMESTAMP, 16). %(1 << 4) --define(SEQ_TRACE_MON_TIMESTAMP, 32). %(1 << 5) +%% Don't forget to update seq_trace_SUITE after changing these. +-define(SEQ_TRACE_SEND, 1). %(1 << 0) +-define(SEQ_TRACE_RECEIVE, 2). %(1 << 1) +-define(SEQ_TRACE_PRINT, 4). %(1 << 2) +-define(SEQ_TRACE_NOW_TIMESTAMP, 8). %(1 << 3) +-define(SEQ_TRACE_STRICT_MON_TIMESTAMP, 16). %(1 << 4) +-define(SEQ_TRACE_MON_TIMESTAMP, 32). %(1 << 5) +-define(SEQ_TRACE_SPAWN, 64). %(1 << 6) -export([set_token/1, set_token/2, @@ -39,7 +41,8 @@ %%--------------------------------------------------------------------------- --type flag() :: 'send' | 'receive' | 'print' | 'timestamp' | 'monotonic_timestamp' | 'strict_monotonic_timestamp'. +-type flag() :: 'send' | 'spawn' | 'receive' | 'print' | 'timestamp' | + 'monotonic_timestamp' | 'strict_monotonic_timestamp'. -type component() :: 'label' | 'serial' | flag(). -type value() :: (Label :: term()) | {Previous :: non_neg_integer(), @@ -142,10 +145,11 @@ set_token2([]) -> decode_flags(Flags) -> Print = (Flags band ?SEQ_TRACE_PRINT) > 0, Send = (Flags band ?SEQ_TRACE_SEND) > 0, + Spawn = (Flags band ?SEQ_TRACE_SPAWN) > 0, Rec = (Flags band ?SEQ_TRACE_RECEIVE) > 0, NowTs = (Flags band ?SEQ_TRACE_NOW_TIMESTAMP) > 0, StrictMonTs = (Flags band ?SEQ_TRACE_STRICT_MON_TIMESTAMP) > 0, MonTs = (Flags band ?SEQ_TRACE_MON_TIMESTAMP) > 0, - [{print,Print},{send,Send},{'receive',Rec},{timestamp,NowTs}, + [{print,Print},{send,Send},{spawn,Spawn},{'receive',Rec},{timestamp,NowTs}, {strict_monotonic_timestamp, StrictMonTs}, {monotonic_timestamp, MonTs}]. diff --git a/lib/kernel/src/user.erl b/lib/kernel/src/user.erl index 5a3487a9ba..81520dd841 100644 --- a/lib/kernel/src/user.erl +++ b/lib/kernel/src/user.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2017. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -537,54 +537,6 @@ get_line_doit(Prompt, Port, Q, Accu, Enc) -> binrev(L, T) -> list_to_binary(lists:reverse(L, T)). -%% is_cr_at(Pos,Bin) -> -%% case Bin of -%% <<_:Pos/binary,$\r,_/binary>> -> -%% true; -%% _ -> -%% false -%% end. - -%% collect_line_bin_re(Bin,_Data,Stack,_) -> -%% case re:run(Bin,<<"\n">>) of -%% nomatch -> -%% X = byte_size(Bin)-1, -%% case is_cr_at(X,Bin) of -%% true -> -%% <<D:X/binary,_/binary>> = Bin, -%% [<<$\r>>,D|Stack]; -%% false -> -%% [Bin|Stack] -%% end; -%% {match,[{Pos,1}]} -> -%% PosPlus = Pos + 1, -%% case Stack of -%% [] -> -%% case is_cr_at(Pos - 1,Bin) of -%% false -> -%% <<Head:PosPlus/binary,Tail/binary>> = Bin, -%% {stop, Head, Tail}; -%% true -> -%% PosMinus = Pos - 1, -%% <<Head:PosMinus/binary,_,_,Tail/binary>> = Bin, -%% {stop, binrev([],[Head,$\n]),Tail} -%% end; -%% [<<$\r>>|Stack1] when Pos =:= 0 -> - -%% <<_:PosPlus/binary,Tail/binary>> = Bin, -%% {stop,binrev(Stack1, [$\n]),Tail}; -%% _ -> -%% case is_cr_at(Pos - 1,Bin) of -%% false -> -%% <<Head:PosPlus/binary,Tail/binary>> = Bin, -%% {stop,binrev(Stack, [Head]),Tail}; -%% true -> -%% PosMinus = Pos - 1, -%% <<Head:PosMinus/binary,_,_,Tail/binary>> = Bin, -%% {stop, binrev(Stack,[Head,$\n]),Tail} -%% end -%% end -%% end. %% get_chars(Prompt, Module, Function, XtraArg, Port, Queue, Encoding) %% Gets characters from the input port until the applied function %% returns {stop,Result,RestBuf}. Does not block output until input @@ -618,9 +570,6 @@ get_chars(Prompt, M, F, Xa, Port, Q, State, Enc) -> {Port, eof} -> put(eof, true), {ok, eof, queue:new()}; - %%{io_request,From,ReplyAs,Request} when is_pid(From) -> - %% get_chars_req(Prompt, M, F, Xa, Port, queue:new(), State, - %% Request, From, ReplyAs); {io_request,From,ReplyAs,{get_geometry,_}=Req} when is_pid(From) -> do_io_request(Req, From, ReplyAs, Port, queue:new()), %Keep Q over this call diff --git a/lib/kernel/test/erl_distribution_wb_SUITE.erl b/lib/kernel/test/erl_distribution_wb_SUITE.erl index 8256444bdc..bb42a0ac39 100644 --- a/lib/kernel/test/erl_distribution_wb_SUITE.erl +++ b/lib/kernel/test/erl_distribution_wb_SUITE.erl @@ -47,6 +47,9 @@ R end). +-define(EPMD_DIST_HIGH, 6). +-define(EPMD_DIST_LOW, 5). + -define(DFLAG_PUBLISHED,1). -define(DFLAG_ATOM_CACHE,2). -define(DFLAG_EXTENDED_REFERENCES,4). @@ -57,15 +60,18 @@ -define(DFLAG_NEW_FUN_TAGS,16#80). -define(DFLAG_EXTENDED_PIDS_PORTS,16#100). -define(DFLAG_UTF8_ATOMS, 16#10000). +-define(DFLAG_BIG_CREATION, 16#40000). %% From R9 and forward extended references is compulsory %% From R10 and forward extended pids and ports are compulsory %% From R20 and forward UTF8 atoms are compulsory %% From R21 and forward NEW_FUN_TAGS is compulsory (no more tuple fallback {fun, ...}) +%% From R23 and forward BIG_CREATION is compulsory -define(COMPULSORY_DFLAGS, (?DFLAG_EXTENDED_REFERENCES bor ?DFLAG_EXTENDED_PIDS_PORTS bor ?DFLAG_UTF8_ATOMS bor - ?DFLAG_NEW_FUN_TAGS)). + ?DFLAG_NEW_FUN_TAGS bor + ?DFLAG_BIG_CREATION)). -define(PASS_THROUGH, $p). @@ -208,9 +214,9 @@ pending_up_md5(Node,OurName,Cookie) -> {ok, SocketA} = gen_tcp:connect(atom_to_list(NB),PortNo, [{active,false}, {packet,2}]), - send_name(SocketA,OurName,5), + send_name(SocketA,OurName, ?EPMD_DIST_HIGH), ok = recv_status(SocketA), - {hidden,Node,5,HisChallengeA} = recv_challenge(SocketA), % See 1) + {hidden,Node,?EPMD_DIST_HIGH,HisChallengeA} = recv_challenge(SocketA), % See 1) OurChallengeA = gen_challenge(), OurDigestA = gen_digest(HisChallengeA, Cookie), send_challenge_reply(SocketA, OurChallengeA, OurDigestA), @@ -224,11 +230,11 @@ pending_up_md5(Node,OurName,Cookie) -> {ok, SocketB} = gen_tcp:connect(atom_to_list(NB),PortNo, [{active,false}, {packet,2}]), - send_name(SocketB,OurName,5), + send_name(SocketB,OurName, ?EPMD_DIST_HIGH), alive = recv_status(SocketB), send_status(SocketB, true), gen_tcp:close(SocketA), - {hidden,Node,5,HisChallengeB} = recv_challenge(SocketB), % See 1) + {hidden,Node,?EPMD_DIST_HIGH,HisChallengeB} = recv_challenge(SocketB), % See 1) OurChallengeB = gen_challenge(), OurDigestB = gen_digest(HisChallengeB, Cookie), send_challenge_reply(SocketB, OurChallengeB, OurDigestB), @@ -254,7 +260,7 @@ simultaneous_md5(Node, OurName, Cookie) when OurName < Node -> Else -> exit(Else) end, - EpmdSocket = register(OurName, LSocket, 1, 5), + EpmdSocket = register_node(OurName, LSocket, ?EPMD_DIST_LOW, ?EPMD_DIST_HIGH), {NA, NB} = split(Node), rpc:cast(Node, net_adm, ping, [OurName]), receive after 1000 -> ok end, @@ -262,7 +268,7 @@ simultaneous_md5(Node, OurName, Cookie) when OurName < Node -> {ok, SocketA} = gen_tcp:connect(atom_to_list(NB),PortNo, [{active,false}, {packet,2}]), - send_name(SocketA,OurName,5), + send_name(SocketA,OurName, ?EPMD_DIST_HIGH), %% We are still not marked up on the other side, as our first message %% is not sent. SocketB = case gen_tcp:accept(LSocket) of @@ -275,11 +281,11 @@ simultaneous_md5(Node, OurName, Cookie) when OurName < Node -> %% Now we are expected to close A gen_tcp:close(SocketA), %% But still Socket B will continue - {normal,Node,5} = recv_name(SocketB), % See 1) + {normal,Node,?EPMD_DIST_HIGH} = recv_name(SocketB), % See 1) send_status(SocketB, ok_simultaneous), MyChallengeB = gen_challenge(), - send_challenge(SocketB, OurName, MyChallengeB,5), - HisChallengeB = recv_challenge_reply(SocketB, MyChallengeB, Cookie), + send_challenge(SocketB, OurName, MyChallengeB, ?EPMD_DIST_HIGH), + {ok,HisChallengeB} = recv_challenge_reply(SocketB, MyChallengeB, Cookie), DigestB = gen_digest(HisChallengeB,Cookie), send_challenge_ack(SocketB, DigestB), inet:setopts(SocketB, [{active, false}, @@ -301,7 +307,8 @@ simultaneous_md5(Node, OurName, Cookie) when OurName > Node -> Else -> exit(Else) end, - EpmdSocket = register(OurName, LSocket, 1, 5), + EpmdSocket = register_node(OurName, LSocket, + ?EPMD_DIST_LOW, ?EPMD_DIST_HIGH), {NA, NB} = split(Node), rpc:cast(Node, net_adm, ping, [OurName]), receive after 1000 -> ok end, @@ -315,16 +322,16 @@ simultaneous_md5(Node, OurName, Cookie) when OurName > Node -> Else2 -> exit(Else2) end, - send_name(SocketA,OurName,5), + send_name(SocketA,OurName, ?EPMD_DIST_HIGH), ok_simultaneous = recv_status(SocketA), %% Socket B should die during this case catch begin - {normal,Node,5} = recv_name(SocketB), % See 1) + {normal,Node,?EPMD_DIST_HIGH} = recv_name(SocketB), % See 1) send_status(SocketB, ok_simultaneous), MyChallengeB = gen_challenge(), send_challenge(SocketB, OurName, MyChallengeB, 5), - HisChallengeB = recv_challenge_reply( + {ok,HisChallengeB} = recv_challenge_reply( SocketB, MyChallengeB, Cookie), @@ -346,7 +353,7 @@ simultaneous_md5(Node, OurName, Cookie) when OurName > Node -> end, gen_tcp:close(SocketB), %% But still Socket A will continue - {hidden,Node,5,HisChallengeA} = recv_challenge(SocketA), % See 1) + {hidden,Node,?EPMD_DIST_HIGH,HisChallengeA} = recv_challenge(SocketA), % See 1) OurChallengeA = gen_challenge(), OurDigestA = gen_digest(HisChallengeA, Cookie), send_challenge_reply(SocketA, OurChallengeA, OurDigestA), @@ -372,7 +379,7 @@ missing_compulsory_dflags(Config) when is_list(Config) -> [{active,false}, {packet,2}]), BadNode = list_to_atom(atom_to_list(Name2)++"@"++atom_to_list(NB)), - send_name(SocketA,BadNode,5,0), + send_name(SocketA,BadNode, ?EPMD_DIST_HIGH, 0), not_allowed = recv_status(SocketA), gen_tcp:close(SocketA), stop_node(Node), @@ -516,16 +523,16 @@ send_challenge_reply(Socket, Challenge, Digest) -> recv_challenge_reply(Socket, ChallengeA, Cookie) -> case gen_tcp:recv(Socket, 0) of - {ok,[$r,CB3,CB2,CB1,CB0 | SumB]} when length(SumB) == 16 -> + {ok,[$r,CB3,CB2,CB1,CB0 | SumB]=Data} when length(SumB) == 16 -> SumA = gen_digest(ChallengeA, Cookie), ChallengeB = ?u32(CB3,CB2,CB1,CB0), if SumB == SumA -> - ChallengeB; + {ok,ChallengeB}; true -> - ?shutdown(bad_challenge_reply) + {error,Data} end; - _ -> - ?shutdown(no_node) + Err -> + {error,Err} end. send_challenge_ack(Socket, Digest) -> @@ -620,6 +627,13 @@ wait_for_reg_reply(Socket, SoFar) -> receive {tcp, Socket, Data0} -> case SoFar ++ Data0 of + [$v, Result, A, B, C, D] -> + case Result of + 0 -> + {alive, Socket, ?u32(A, B, C, D)}; + _ -> + {error, duplicate_name} + end; [$y, Result, A, B] -> case Result of 0 -> @@ -640,7 +654,7 @@ wait_for_reg_reply(Socket, SoFar) -> end. -register(NodeName, ListenSocket, VLow, VHigh) -> +register_node(NodeName, ListenSocket, VLow, VHigh) -> {ok,{_,TcpPort}} = inet:sockname(ListenSocket), case do_register_node(NodeName, TcpPort, VLow, VHigh) of {alive, Socket, _Creation} -> diff --git a/lib/kernel/test/file_SUITE.erl b/lib/kernel/test/file_SUITE.erl index 1c1b35abc1..e4299cd346 100644 --- a/lib/kernel/test/file_SUITE.erl +++ b/lib/kernel/test/file_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2018. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -58,6 +58,8 @@ -export([ file_info_basic_file/1, file_info_basic_directory/1, file_info_bad/1, file_info_times/1, file_write_file_info/1, file_wfi_helpers/1]). +-export([ file_handle_info_basic_file/1, file_handle_info_basic_directory/1, + file_handle_info_times/1]). -export([rename/1, access/1, truncate/1, datasync/1, sync/1, read_write/1, pread_write/1, append/1, exclusive/1]). -export([ e_delete/1, e_rename/1, e_make_dir/1, e_del_dir/1]). @@ -155,7 +157,10 @@ groups() -> {pos, [], [pos1, pos2, pos3]}, {file_info, [], [file_info_basic_file, file_info_basic_directory, - file_info_bad, file_info_times, file_write_file_info, + file_info_bad, file_info_times, + file_handle_info_basic_file, file_handle_info_basic_directory, + file_handle_info_times, + file_write_file_info, file_wfi_helpers]}, {consult, [], [consult1, path_consult]}, {eval, [], [eval1, path_eval]}, @@ -275,11 +280,11 @@ mini_server(Parent) -> Parent ! {io_request,From,To,{put_chars,Data}}, From ! {io_reply, To, ok}, mini_server(Parent); - {io_request,From,To,{get_chars,'',N}} -> + {io_request,From,To,{get_chars,_Encoding,'',N}} -> Parent ! {io_request,From,To,{get_chars,'',N}}, From ! {io_reply, To, {ok, lists:duplicate(N,$a)}}, mini_server(Parent); - {io_request,From,To,{get_line,''}} -> + {io_request,From,To,{get_line,_Encoding,''}} -> Parent ! {io_request,From,To,{get_line,''}}, From ! {io_reply, To, {ok, "hej\n"}}, mini_server(Parent) @@ -989,6 +994,14 @@ new_modes(Config) when is_list(Config) -> ok end, + % open directory + {ok, Fd9} = ?FILE_MODULE:open(NewDir, [directory]), + ok = ?FILE_MODULE:close(Fd9), + + % open raw directory + {ok, Fd10} = ?FILE_MODULE:open(NewDir, [raw, directory]), + ok = ?FILE_MODULE:close(Fd10), + [] = flush(), ok. @@ -1238,6 +1251,9 @@ open_errors(Config) when is_list(Config) -> {error, E4} = ?FILE_MODULE:open(DataDirSlash, [write]), {eisdir,eisdir,eisdir,eisdir} = {E1,E2,E3,E4}, + Real = filename:join(DataDir, "realmen.html"), + {error, enotdir} = ?FILE_MODULE:open(Real, [directory]), + [] = flush(), ok. @@ -1408,7 +1424,8 @@ file_info_basic_directory(Config) when is_list(Config) -> {win32, _} -> test_directory("/", read_write), test_directory("c:/", read_write), - test_directory("c:\\", read_write); + test_directory("c:\\", read_write), + test_directory("\\\\localhost\\c$", read_write); _ -> test_directory("/", read) end, @@ -1540,6 +1557,180 @@ filter_atime(Atime, Config) -> Atime end. +%% Test read_file_info on I/O devices. + +file_handle_info_basic_file(Config) when is_list(Config) -> + RootDir = proplists:get_value(priv_dir, Config), + + %% Create a short file. + Name = filename:join(RootDir, + atom_to_list(?MODULE) + ++"_basic_test.fil"), + {ok,Fd1} = ?FILE_MODULE:open(Name, write), + io:put_chars(Fd1, "foo bar"), + ok = ?FILE_MODULE:close(Fd1), + + %% Test that the file has the expected attributes. + %% The times are tricky, so we will save them to a separate test case. + + {ok, Fd} = ?FILE_MODULE:open(Name, read), + {ok,FileInfo} = ?FILE_MODULE:read_file_info(Fd), + ok = ?FILE_MODULE:close(Fd), + + {ok, FdRaw} = ?FILE_MODULE:open(Name, [read, raw]), + {ok,FileInfoRaw} = ?FILE_MODULE:read_file_info(FdRaw), + ok = ?FILE_MODULE:close(FdRaw), + + #file_info{size=Size,type=Type,access=Access, + atime=AccessTime,mtime=ModifyTime} = FileInfo = FileInfoRaw, + io:format("Access ~p, Modify ~p", [AccessTime, ModifyTime]), + Size = 7, + Type = regular, + read_write = Access, + true = abs(time_dist(filter_atime(AccessTime, Config), + filter_atime(ModifyTime, + Config))) < 5, + all_integers(tuple_to_list(AccessTime) ++ tuple_to_list(ModifyTime)), + + [] = flush(), + ok. + +file_handle_info_basic_directory(Config) when is_list(Config) -> + %% Note: filename:join/1 removes any trailing slash, + %% which is essential for ?FILE_MODULE:file_info/1 to work on + %% platforms such as Windows95. + RootDir = filename:join([proplists:get_value(priv_dir, Config)]), + + %% Test that the RootDir directory has the expected attributes. + test_directory_handle(RootDir, read_write), + + %% Note that on Windows file systems, + %% "/" or "c:/" are *NOT* directories. + %% Therefore, test that ?FILE_MODULE:file_info/1 behaves as if they were + %% directories. + case os:type() of + {win32, _} -> + test_directory_handle("/", read_write), + test_directory_handle("c:/", read_write), + test_directory_handle("c:\\", read_write), + test_directory_handle("\\\\localhost\\c$", read_write); + _ -> + test_directory_handle("/", read) + end, + ok. + +test_directory_handle(Name, ExpectedAccess) -> + {ok, DirFd} = file:open(Name, [read, directory]), + try + {ok,FileInfo} = ?FILE_MODULE:read_file_info(DirFd), + {ok,FileInfo} = ?FILE_MODULE:read_file_info(DirFd, [raw]), + #file_info{size=Size,type=Type,access=Access, + atime=AccessTime,mtime=ModifyTime} = FileInfo, + io:format("Testing directory ~s", [Name]), + io:format("Directory size is ~p", [Size]), + io:format("Access ~p", [Access]), + io:format("Access time ~p; Modify time~p", + [AccessTime, ModifyTime]), + Type = directory, + Access = ExpectedAccess, + all_integers(tuple_to_list(AccessTime) ++ tuple_to_list(ModifyTime)), + [] = flush(), + ok + after + file:close(DirFd) + end. + +%% Test that the file times behave as they should. + +file_handle_info_times(Config) when is_list(Config) -> + %% We have to try this twice, since if the test runs across the change + %% of a month the time diff calculations will fail. But it won't happen + %% if you run it twice in succession. + test_server:m_out_of_n( + 1,2, + fun() -> file_handle_info_int(Config) end), + ok. + +file_handle_info_int(Config) -> + %% Note: filename:join/1 removes any trailing slash, + %% which is essential for ?FILE_MODULE:file_info/1 to work on + %% platforms such as Windows95. + + RootDir = filename:join([proplists:get_value(priv_dir, Config)]), + io:format("RootDir = ~p", [RootDir]), + + Name = filename:join(RootDir, + atom_to_list(?MODULE) + ++"_file_info.fil"), + {ok,Fd1} = ?FILE_MODULE:open(Name, write), + io:put_chars(Fd1,"foo"), + {ok,FileInfo1} = ?FILE_MODULE:read_file_info(Fd1), + ok = ?FILE_MODULE:close(Fd1), + + {ok,Fd1Raw} = ?FILE_MODULE:open(Name, [read, raw]), + {ok,FileInfo1Raw} = ?FILE_MODULE:read_file_info(Fd1Raw), + ok = ?FILE_MODULE:close(Fd1Raw), + + %% We assert that everything but the size is the same, on some OSs the + %% size may not have been flushed to disc and we do not want to do a + %% sync to force it. + FileInfo1Raw = FileInfo1#file_info{ size = FileInfo1Raw#file_info.size }, + + #file_info{type=regular,atime=AccTime1,mtime=ModTime1} = FileInfo1, + + Now = erlang:localtime(), %??? + io:format("Now ~p",[Now]), + io:format("Open file Acc ~p Mod ~p",[AccTime1,ModTime1]), + true = abs(time_dist(filter_atime(Now, Config), + filter_atime(AccTime1, + Config))) < 8, + true = abs(time_dist(Now,ModTime1)) < 8, + + %% Sleep until we can be sure the seconds value has changed. + %% Note: FAT-based filesystem (like on Windows 95) have + %% a resolution of 2 seconds. + timer:sleep(2200), + + %% close the file, and watch the modify date change + + {ok,Fd2} = ?FILE_MODULE:open(Name, read), + {ok,FileInfo2} = ?FILE_MODULE:read_file_info(Fd2), + ok = ?FILE_MODULE:close(Fd2), + + {ok,Fd2Raw} = ?FILE_MODULE:open(Name, [read, raw]), + {ok,FileInfo2Raw} = ?FILE_MODULE:read_file_info(Fd2Raw), + ok = ?FILE_MODULE:close(Fd2Raw), + + #file_info{size=Size,type=regular,access=Access, + atime=AccTime2,mtime=ModTime2} = FileInfo2 = FileInfo2Raw, + io:format("Closed file Acc ~p Mod ~p",[AccTime2,ModTime2]), + true = time_dist(ModTime1,ModTime2) >= 0, + + %% this file is supposed to be binary, so it'd better keep it's size + Size = 3, + Access = read_write, + + %% Do some directory checking + + {ok,Fd3} = ?FILE_MODULE:open(RootDir, [read, directory]), + {ok,FileInfo3} = ?FILE_MODULE:read_file_info(Fd3), + ok = ?FILE_MODULE:close(Fd3), + + {ok,Fd3Raw} = ?FILE_MODULE:open(RootDir, [read, directory, raw]), + {ok,FileInfo3Raw} = ?FILE_MODULE:read_file_info(Fd3Raw), + ok = ?FILE_MODULE:close(Fd3Raw), + + #file_info{size=DSize,type=directory,access=DAccess, + atime=AccTime3,mtime=ModTime3} = FileInfo3 = FileInfo3Raw, + %% this dir was modified only a few secs ago + io:format("Dir Acc ~p; Mod ~p; Now ~p", [AccTime3, ModTime3, Now]), + true = abs(time_dist(Now,ModTime3)) < 5, + DAccess = read_write, + io:format("Dir size is ~p",[DSize]), + + [] = flush(), + ok. + %% Test the write_file_info/2 function. file_write_file_info(Config) when is_list(Config) -> diff --git a/lib/kernel/test/gen_tcp_api_SUITE.erl b/lib/kernel/test/gen_tcp_api_SUITE.erl index 1be016444f..00c9dc5ed5 100644 --- a/lib/kernel/test/gen_tcp_api_SUITE.erl +++ b/lib/kernel/test/gen_tcp_api_SUITE.erl @@ -594,10 +594,13 @@ unused_ip() -> io:format("we = ~p, unused_ip = ~p~n", [Hent, IP]), IP. -unused_ip(_, _, _, 255) -> error; +unused_ip(255, 255, 255, 255) -> error; +unused_ip(255, B, C, D) -> unused_ip(1, B + 1, C, D); +unused_ip(A, 255, C, D) -> unused_ip(A, 1, C + 1, D); +unused_ip(A, B, 255, D) -> unused_ip(A, B, 1, D + 1); unused_ip(A, B, C, D) -> case inet:gethostbyaddr({A, B, C, D}) of - {ok, _} -> unused_ip(A, B, C, D+1); + {ok, _} -> unused_ip(A + 1, B, C, D); {error, _} -> {ok, {A, B, C, D}} end. diff --git a/lib/kernel/test/interactive_shell_SUITE.erl b/lib/kernel/test/interactive_shell_SUITE.erl index 173e25c520..883abda067 100644 --- a/lib/kernel/test/interactive_shell_SUITE.erl +++ b/lib/kernel/test/interactive_shell_SUITE.erl @@ -22,7 +22,7 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, get_columns_and_rows/1, exit_initial/1, job_control_local/1, - job_control_remote/1, + job_control_remote/1,stop_during_init/1, job_control_remote_noshell/1,ctrl_keys/1, get_columns_and_rows_escript/1]). @@ -44,7 +44,7 @@ all() -> [get_columns_and_rows_escript,get_columns_and_rows, exit_initial, job_control_local, job_control_remote, job_control_remote_noshell, - ctrl_keys]. + ctrl_keys, stop_during_init]. groups() -> []. @@ -205,6 +205,22 @@ exit_initial(Config) when is_list(Config) -> {getline_re,"35"}],[]) end. +stop_during_init(Config) when is_list(Config) -> + case get_progs() of + {error,_Reason} -> + {skip,"No runerl present"}; + {RunErl,_ToErl,Erl} -> + case create_tempdir() of + {error, Reason2} -> + {skip, Reason2}; + Tempdir -> + XArg = " -kernel shell_history true -s init stop", + start_runerl_command(RunErl, Tempdir, "\\\""++Erl++"\\\""++XArg), + {ok, Binary} = file:read_file(filename:join(Tempdir, "erlang.log.1")), + nomatch = binary:match(Binary, <<"*** ERROR: Shell process terminated! ***">>) + end + end. + %% Tests that local shell can be started by means of job control. job_control_local(Config) when is_list(Config) -> case proplists:get_value(default_shell,Config) of @@ -656,10 +672,10 @@ start_runerl_node(RunErl,Erl,Tempdir,Nodename) -> end)++ " -setcookie "++atom_to_list(erlang:get_cookie()) end, - spawn(fun() -> - os:cmd("\""++RunErl++"\" "++Tempdir++"/ "++Tempdir++" \""++ - Erl++XArg++"\"") - end). + spawn(fun() -> start_runerl_command(RunErl, Tempdir, Erl++XArg) end). + +start_runerl_command(RunErl, Tempdir, Cmd) -> + os:cmd("\""++RunErl++"\" "++Tempdir++"/ "++Tempdir++" \""++Cmd++"\""). start_toerl_server(ToErl,Tempdir) -> Pid = spawn(?MODULE,toerl_server,[self(),ToErl,Tempdir]), diff --git a/lib/kernel/test/seq_trace_SUITE.erl b/lib/kernel/test/seq_trace_SUITE.erl index 83a94ab087..adbcef955c 100644 --- a/lib/kernel/test/seq_trace_SUITE.erl +++ b/lib/kernel/test/seq_trace_SUITE.erl @@ -30,7 +30,7 @@ send/1, distributed_send/1, recv/1, distributed_recv/1, trace_exit/1, distributed_exit/1, call/1, port/1, match_set_seq_token/1, gc_seq_token/1, label_capability_mismatch/1, - send_literal/1]). + send_literal/1,inherit_on_spawn/1,spawn_flag/1]). %% internal exports -export([simple_tracer/2, one_time_receiver/0, one_time_receiver/1, @@ -53,7 +53,8 @@ all() -> distributed_send, recv, distributed_recv, trace_exit, old_heap_token, distributed_exit, call, port, match_set_seq_token, - gc_seq_token, label_capability_mismatch]. + gc_seq_token, label_capability_mismatch, + inherit_on_spawn, spawn_flag]. groups() -> []. @@ -83,14 +84,29 @@ token_set_get(Config) when is_list(Config) -> do_token_set_get(timestamp), do_token_set_get(monotonic_timestamp), do_token_set_get(strict_monotonic_timestamp). - + +-define(SEQ_TRACE_SEND, 1). %(1 << 0) +-define(SEQ_TRACE_RECEIVE, 2). %(1 << 1) +-define(SEQ_TRACE_PRINT, 4). %(1 << 2) +-define(SEQ_TRACE_NOW_TIMESTAMP, 8). %(1 << 3) +-define(SEQ_TRACE_STRICT_MON_TIMESTAMP, 16). %(1 << 4) +-define(SEQ_TRACE_MON_TIMESTAMP, 32). %(1 << 5) +-define(SEQ_TRACE_SPAWN, 64). %(1 << 6) + do_token_set_get(TsType) -> - io:format("Testing ~p~n", [TsType]), + BaseOpts = ?SEQ_TRACE_SEND bor + ?SEQ_TRACE_RECEIVE bor + ?SEQ_TRACE_PRINT bor + ?SEQ_TRACE_SPAWN, Flags = case TsType of - timestamp -> 15; - strict_monotonic_timestamp -> 23; - monotonic_timestamp -> 39 - end, + timestamp -> + BaseOpts bor ?SEQ_TRACE_NOW_TIMESTAMP; + strict_monotonic_timestamp -> + BaseOpts bor ?SEQ_TRACE_STRICT_MON_TIMESTAMP; + monotonic_timestamp -> + BaseOpts bor ?SEQ_TRACE_MON_TIMESTAMP + end, + ct:pal("Type ~p, flags = ~p~n", [TsType, Flags]), Self = self(), seq_trace:reset_trace(), %% Test that initial seq_trace is disabled @@ -102,6 +118,8 @@ do_token_set_get(TsType) -> {print,true} = seq_trace:get_token(print), false = seq_trace:set_token(send,true), {send,true} = seq_trace:get_token(send), + false = seq_trace:set_token(spawn,true), + {spawn,true} = seq_trace:get_token(spawn), false = seq_trace:set_token('receive',true), {'receive',true} = seq_trace:get_token('receive'), false = seq_trace:set_token(TsType,true), @@ -466,8 +484,6 @@ call(Config) when is_list(Config) -> 1 = erlang:trace(Self, true, [call, set_on_spawn, {tracer, TrB(pid)}]), - Label = 17, - seq_trace:set_token(label, Label), % Token enters here!! RefB = make_ref(), Pid2B = spawn_link( fun() -> @@ -481,6 +497,12 @@ call(Config) when is_list(Config) -> RefB = call_tracee_1(RefB), Pid2B ! {self(), msg, RefB} end), + + %% The token is set *AFTER* spawning to make sure we're testing that the + %% token follows on send and not that it inherits on spawn. + Label = 17, + seq_trace:set_token(label, Label), + Pid1B ! {Self, msg, RefB}, %% The message is passed Self -> Pid1B -> Pid2B -> Self, and the %% seq_trace token follows invisibly. Traced functions are @@ -501,6 +523,62 @@ call(Config) when is_list(Config) -> seq_trace:reset_trace(), ok. +%% The token should follow spawn, just like it follows messages. +inherit_on_spawn(Config) when is_list(Config) -> + seq_trace:reset_trace(), + start_tracer(), + + Ref = make_ref(), + seq_trace:set_token(label,Ref), + set_token_flags([send]), + + Self = self(), + Other = spawn(fun() -> Self ! {gurka,Ref} end), + + receive {gurka,Ref} -> ok end, + seq_trace:reset_trace(), + + [{Ref,{send,_,Other,Self,{gurka,Ref}}, _Ts}] = stop_tracer(1), + + ok. + +spawn_flag(Config) when is_list(Config) -> + seq_trace:reset_trace(), + start_tracer(), + + Ref = make_ref(), + seq_trace:set_token(label,Ref), + set_token_flags([spawn]), + + Self = self(), + + {serial,{0,0}} = seq_trace:get_token(serial), + + %% The serial number is bumped on spawning (just like message passing), so + %% our child should inherit a counter of 1. + ProcessA = spawn(fun() -> + {serial,{0,1}} = seq_trace:get_token(serial), + Self ! {a,Ref} + end), + receive {a,Ref} -> ok end, + + {serial,{1,2}} = seq_trace:get_token(serial), + + ProcessB = spawn(fun() -> + {serial,{2,3}} = seq_trace:get_token(serial), + Self ! {b,Ref} + end), + receive {b,Ref} -> ok end, + + {serial,{3,4}} = seq_trace:get_token(serial), + + seq_trace:reset_trace(), + + [{Ref,{spawn,{0,1},Self,ProcessA,[]}, _Ts}, + {Ref,{spawn,{2,3},Self,ProcessB,[]}, _Ts}] = stop_tracer(2), + + ok. + %% Send trace messages to a port. port(Config) when is_list(Config) -> lists:foreach(fun (TsType) -> do_port(TsType, Config) end, @@ -938,7 +1016,7 @@ stop_tracer(N) when is_integer(N) -> receive {tracerlog,Data} -> Data - after 1000 -> + after 5000 -> {error,timeout} end end. diff --git a/lib/megaco/Makefile b/lib/megaco/Makefile index f385df6a5c..624c4b0619 100644 --- a/lib/megaco/Makefile +++ b/lib/megaco/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1999-2019. All Rights Reserved. +# Copyright Ericsson AB 1999-2016. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -110,7 +110,7 @@ DIA_ANALYSIS = $(basename $(DIA_PLT)).dialyzer_analysis # ---------------------------------------------------- include $(ERL_TOP)/make/otp_subdir.mk -.PHONY: reconf conf dconf econf configure setup info version \ +.PHONY: reconf conf dconf econf configure setup info_megaco version \ app_install dialyzer reconf: @@ -136,16 +136,15 @@ configure: configure.in setup: (cd src && $(MAKE) $@) -info: +info_megaco: @echo "APP_RELEASE_DIR: $(APP_RELEASE_DIR)" @echo "APP_DIR: $(APP_DIR)" @echo "APP_TAR_FILE: $(APP_TAR_FILE)" @echo "OTP_INSTALL_DIR: $(OTP_INSTALL_DIR)" @echo "APP_INSTALL_DIR: $(APP_INSTALL_DIR)" @echo "" - @echo "DIA_PLT: $(DIA_PLT)" - @echo "DIA_ANALYSIS: $(DIA_ANALYSIS)" - @echo "" + +info: info_megaco version: @echo "$(VSN)" @@ -204,25 +203,6 @@ tar: $(APP_TAR_FILE) $(APP_TAR_FILE): $(APP_DIR) (cd "$(APP_RELEASE_DIR)"; gtar zcf "$(subst $(space),\ ,$@)" $(DIR_NAME)) -dialyzer_plt: $(DIA_PLT) - -$(DIA_PLT): Makefile - @echo "Building $(APPLICATION) plt file" - @dialyzer --build_plt \ - --output_plt $@ \ - -r ../$(APPLICATION)/ebin \ - ../../lib/kernel/ebin \ - ../../lib/stdlib/ebin \ - ../../lib/runtime_tools/ebin \ - ../../lib/asn1/ebin \ - ../../lib/debugger/ebin \ - ../../lib/et/ebin \ - ../../erts/preloaded/ebin \ - --output $(DIA_ANALYSIS) \ - --verbose - -dialyzer: $(DIA_PLT) - @echo "Running dialyzer on $(APPLICATION)" - @dialyzer --plt $< \ - ../$(APPLICATION)/ebin \ - --verbose +DIA_PLT_APPS=asn1 runtime_tools et debugger + +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/megaco/test/megaco_mess_test.erl b/lib/megaco/test/megaco_mess_test.erl index 46aeabd679..0486c13e5f 100644 --- a/lib/megaco/test/megaco_mess_test.erl +++ b/lib/megaco/test/megaco_mess_test.erl @@ -13227,8 +13227,8 @@ otp_8183_r1_mgc_reply_msg(Mid, TransId, CR, Cid) -> {?MODULE, otp_8183_r1_mg_verify_handle_connect, []}). -define(otp_8183_r1_mg_verify_service_change_rep_fun(), {?MODULE, otp_8183_r1_mg_verify_service_change_rep, []}). --define(otp_8183_r1_mg_verify_notify_rep_fun(Nr - {?MODULE, otp_8183_r1_mg_verify_notify_rep, [Nr). +-define(otp_8183_r1_mg_verify_notify_rep_fun(Nr), + {?MODULE, otp_8183_r1_mg_verify_notify_rep, [Nr]}). -else. -define(otp_8183_r1_mg_verify_handle_connect_fun(), otp_8183_r1_mg_verify_handle_connect_fun()). diff --git a/lib/mnesia/Makefile b/lib/mnesia/Makefile index 810433c4d0..d0edd48af9 100644 --- a/lib/mnesia/Makefile +++ b/lib/mnesia/Makefile @@ -38,3 +38,4 @@ SPECIAL_TARGETS = # ---------------------------------------------------- include $(ERL_TOP)/make/otp_subdir.mk +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/mnesia/src/mnesia.erl b/lib/mnesia/src/mnesia.erl index 02bc884e36..9d462f0a63 100644 --- a/lib/mnesia/src/mnesia.erl +++ b/lib/mnesia/src/mnesia.erl @@ -2731,7 +2731,7 @@ create_table(Arg) -> create_table(Name, Arg) when is_list(Arg) -> mnesia_schema:create_table([{name, Name}| Arg]); create_table(Name, Arg) -> - {aborted, badarg, Name, Arg}. + {aborted, {badarg, Name, Arg}}. -spec delete_table(Tab::table()) -> t_result('ok'). delete_table(Tab) -> diff --git a/lib/mnesia/src/mnesia_text.erl b/lib/mnesia/src/mnesia_text.erl index cc21621ff4..ee31fdfd27 100644 --- a/lib/mnesia/src/mnesia_text.erl +++ b/lib/mnesia/src/mnesia_text.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2018. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -170,7 +170,7 @@ read_terms(Stream, File, Line, L) -> end. read_term_from_stream(Stream, File, Line) -> - R = io:request(Stream, {get_until,'',erl_scan,tokens,[Line]}), + R = io:request(Stream, {get_until,latin1,'',erl_scan,tokens,[Line]}), case R of {ok,Toks,EndLine} -> case erl_parse:parse_term(Toks) of diff --git a/lib/observer/Makefile b/lib/observer/Makefile index 8483922f76..ffd73051b9 100644 --- a/lib/observer/Makefile +++ b/lib/observer/Makefile @@ -36,4 +36,6 @@ SPECIAL_TARGETS = # include $(ERL_TOP)/make/otp_subdir.mk +DIA_PLT_APPS=runtime_tools et wx +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/observer/test/ttb_SUITE.erl b/lib/observer/test/ttb_SUITE.erl index 33133dd78d..f8bb2e5eb0 100644 --- a/lib/observer/test/ttb_SUITE.erl +++ b/lib/observer/test/ttb_SUITE.erl @@ -658,11 +658,13 @@ seq_trace(Config) when is_list(Config) -> ?line ok = ttb:format( [filename:join(Privdir,atom_to_list(Node)++"-seq_trace")]), ?line [{trace_ts,StartProc,call,{?MODULE,seq,[]},{_,_,_}}, - {seq_trace,0,{send,{0,1},StartProc,P1Proc,{Start,P2}}}, - {seq_trace,0,{send,{1,2},P1Proc,P2Proc,{P1,Start}}}, - {seq_trace,0,{send,{2,3},P2Proc,StartProc,{P2,P1}}}, + {seq_trace,0,{send,{First, Seq0},StartProc,P1Proc,{Start,P2}}}, + {seq_trace,0,{send,{Seq0, Seq1},P1Proc,P2Proc,{P1,Start}}}, + {seq_trace,0,{send,{Seq1, Last},P2Proc,StartProc,{P2,P1}}}, end_of_trace] = flush(), - + true = First < Seq0, + true = Seq0 < Seq1, + true = Seq1 < Last, %% Additional test for metatrace case StartProc of {Start,_,_} -> ok; diff --git a/lib/odbc/Makefile b/lib/odbc/Makefile index f7816c25fc..ee39992a84 100644 --- a/lib/odbc/Makefile +++ b/lib/odbc/Makefile @@ -69,12 +69,6 @@ do_configure: configure configure: configure.in autoconf -.PHONY: info - -info: - @echo "ODBC_VSN: $(ODBC_VSN)" - - # ---------------------------------------------------- # Application (source) release targets # ---------------------------------------------------- @@ -114,3 +108,5 @@ tar: $(APP_TAR_FILE) $(APP_TAR_FILE): $(APP_DIR) (cd $(APP_RELEASE_DIR); gtar zcf $(APP_TAR_FILE) $(DIR_NAME)) + +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/os_mon/Makefile b/lib/os_mon/Makefile index 40ce94e0c7..f45065a79d 100644 --- a/lib/os_mon/Makefile +++ b/lib/os_mon/Makefile @@ -35,3 +35,4 @@ SPECIAL_TARGETS = # include $(ERL_TOP)/make/otp_subdir.mk +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/os_mon/test/cpu_sup_SUITE.erl b/lib/os_mon/test/cpu_sup_SUITE.erl index ba28f31f26..7a8065c591 100644 --- a/lib/os_mon/test/cpu_sup_SUITE.erl +++ b/lib/os_mon/test/cpu_sup_SUITE.erl @@ -155,46 +155,72 @@ tiny_diff(A, B) -> -define(SPIN_TIME, 1000). +spinner(Parent) -> + receive + stop -> Parent ! stopped + after 0 -> spinner(Parent) + end. + %% Test utilization values util_values(Config) when is_list(Config) -> - + NrOfProcessors = + case erlang:system_info(logical_processors_available) of + unknown -> 2; + X -> X + end, Tester = self(), Ref = make_ref(), - Loop = fun (L) -> L(L) end, Spinner = fun () -> - Looper = spawn_link(fun () -> Loop(Loop) end), + Spinner = self(), + NrOfProcesses = NrOfProcessors, + Loopers = + [spawn_link(fun () -> spinner(Spinner) end) + || _ <- lists:seq(1,NrOfProcesses)], receive after ?SPIN_TIME -> ok end, - unlink(Looper), - exit(Looper, kill), + [begin + Looper ! stop, + receive stopped -> ok end + end + || Looper <- Loopers], Tester ! Ref end, - + Spin = fun () -> + spawn_link(Spinner), + receive Ref -> ok end + end, cpu_sup:util(), - - spawn_link(Spinner), - receive Ref -> ok end, - HighUtil1 = cpu_sup:util(), - receive after ?SPIN_TIME -> ok end, - LowUtil1 = cpu_sup:util(), + LowUtil0 = cpu_sup:util(), + case LowUtil0 of + U when U > ((100.0 / NrOfProcessors) * 0.33) -> + %% We cannot run this test if the system is doing other + %% work at the same time as the result will be unreliable + {skip, io_lib:format("CPU utilization was too high (~f%)", [LowUtil0])}; + _ -> + cpu_sup:util(), + Spin(), + HighUtil1 = cpu_sup:util(), - spawn_link(Spinner), - receive Ref -> ok end, - HighUtil2 = cpu_sup:util(), + receive after ?SPIN_TIME -> ok end, + LowUtil1 = cpu_sup:util(), - receive after ?SPIN_TIME -> ok end, - LowUtil2 = cpu_sup:util(), + Spin(), + HighUtil2 = cpu_sup:util(), - Utils = [{high1,HighUtil1}, {low1,LowUtil1}, - {high2,HighUtil2}, {low2,LowUtil2}], - io:format("Utils: ~p~n", [Utils]), + receive after ?SPIN_TIME -> ok end, + LowUtil2 = cpu_sup:util(), - false = LowUtil1 > HighUtil1, - false = LowUtil1 > HighUtil2, - false = LowUtil2 > HighUtil1, - false = LowUtil2 > HighUtil2, + Utils = [{high1,HighUtil1}, {low1,LowUtil1}, + {high2,HighUtil2}, {low2,LowUtil2}], + io:format("Utils: ~p~n", [Utils]), - ok. + false = LowUtil1 > HighUtil1, + false = LowUtil1 > HighUtil2, + false = LowUtil2 > HighUtil1, + false = LowUtil2 > HighUtil2, + + ok + end. % Outdated diff --git a/lib/parsetools/Makefile b/lib/parsetools/Makefile index e9de5c43cb..2ddb06feb1 100644 --- a/lib/parsetools/Makefile +++ b/lib/parsetools/Makefile @@ -37,3 +37,4 @@ SPECIAL_TARGETS = # ---------------------------------------------------- include $(ERL_TOP)/make/otp_subdir.mk +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/parsetools/doc/src/leex.xml b/lib/parsetools/doc/src/leex.xml index 3b82f60201..3944c650d8 100644 --- a/lib/parsetools/doc/src/leex.xml +++ b/lib/parsetools/doc/src/leex.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2009</year><year>2017</year> + <year>2009</year><year>2019</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -193,7 +193,7 @@ Token = tuple()</code> but used through the i/o system where it can typically be called in an application by:</p> <code> -io:request(InFile, {get_until,Prompt,Module,token,[Line]}) +io:request(InFile, {get_until,unicode,Prompt,Module,token,[Line]}) -> TokenRet</code> </desc> </func> @@ -240,7 +240,7 @@ io:request(InFile, {get_until,Prompt,Module,token,[Line]}) but used through the i/o system where it can typically be called in an application by:</p> <code> -io:request(InFile, {get_until,Prompt,Module,tokens,[Line]}) +io:request(InFile, {get_until,unicode,Prompt,Module,tokens,[Line]}) -> TokensRet</code> </desc> </func> diff --git a/lib/public_key/Makefile b/lib/public_key/Makefile index 7a5c1c1443..ed8901a8af 100644 --- a/lib/public_key/Makefile +++ b/lib/public_key/Makefile @@ -38,3 +38,6 @@ SPECIAL_TARGETS = # ---------------------------------------------------- include $(ERL_TOP)/make/otp_subdir.mk +DIA_PLT_APPS=asn crypto + +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/reltool/Makefile b/lib/reltool/Makefile index 4b6aad07b3..0a39d7daa8 100644 --- a/lib/reltool/Makefile +++ b/lib/reltool/Makefile @@ -36,3 +36,6 @@ SPECIAL_TARGETS = # ---------------------------------------------------- include $(ERL_TOP)/make/otp_subdir.mk +DIA_PLT_APPS=wx sasl + +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/runtime_tools/Makefile b/lib/runtime_tools/Makefile index eec1ff379b..4b0f1633ab 100644 --- a/lib/runtime_tools/Makefile +++ b/lib/runtime_tools/Makefile @@ -36,4 +36,6 @@ SPECIAL_TARGETS = # include $(ERL_TOP)/make/otp_subdir.mk +DIA_PLT_APPS=mnesia +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/runtime_tools/examples/function-calls.d b/lib/runtime_tools/examples/function-calls.d index f8ca388228..a51ff51253 100644 --- a/lib/runtime_tools/examples/function-calls.d +++ b/lib/runtime_tools/examples/function-calls.d @@ -19,39 +19,85 @@ * %CopyrightEnd% */ +/** + * Triggered on local function entry + * + * @param arg0 pid + * @param arg1 MFA of the function + * @param arg2 depth + */ erlang*:::local-function-entry { printf("pid %s enter (local) %s depth %d\n", copyinstr(arg0), copyinstr(arg1), arg2); } +/** + * Triggered on global function entry + * + * @param arg0 pid + * @param arg1 MFA of the function + * @param arg2 depth + */ erlang*:::global-function-entry { printf("pid %s enter (global) %s depth %d\n", copyinstr(arg0), copyinstr(arg1), arg2); } +/** + * Triggered upon function return, either global or + * local + * + * @param arg0 pid + * @param arg1 MFA of the returned from function + * @param arg2 depth + */ erlang*:::function-return { printf("pid %s return %s depth %d\n", copyinstr(arg0), copyinstr(arg1), arg2); } +/** + * Triggered on built-in function entry + * + * @param arg0 pid + * @param arg1 MFA of the function + */ erlang*:::bif-entry { printf("pid %s BIF entry mfa %s\n", copyinstr(arg0), copyinstr(arg1)); } +/** + * Triggered on built-in function return + * + * @param arg0 pid + * @param arg1 MFA of the returned from function + */ erlang*:::bif-return { printf("pid %s BIF return mfa %s\n", copyinstr(arg0), copyinstr(arg1)); } +/** + * Triggered on native function entry + * + * @param arg0 pid + * @param arg1 MFA of the function + */ erlang*:::nif-entry { printf("pid %s NIF entry mfa %s\n", copyinstr(arg0), copyinstr(arg1)); } +/** + * Triggered upon native function return + * + * @param arg0 pid + * @param arg1 MFA of the returned from function + */ erlang*:::nif-return { printf("pid %s NIF return mfa %s\n", copyinstr(arg0), copyinstr(arg1)); diff --git a/lib/runtime_tools/examples/function-calls.systemtap b/lib/runtime_tools/examples/function-calls.systemtap index 6bb173b3ec..8f748ce0d1 100644 --- a/lib/runtime_tools/examples/function-calls.systemtap +++ b/lib/runtime_tools/examples/function-calls.systemtap @@ -29,39 +29,85 @@ * to your environment. */ +/** + * Triggered on local function entry + * + * @param arg0 pid + * @param arg1 MFA of the function + * @param arg2 depth + */ probe process("beam.smp").mark("local-function-entry") { printf("pid %s enter (local) %s depth %d\n", user_string($arg1), user_string($arg2), $arg3); } +/** + * Triggered on global function entry + * + * @param arg0 pid + * @param arg1 MFA of the function + * @param arg2 depth + */ probe process("beam.smp").mark("global-function-entry") { printf("pid %s enter (global) %s depth %d\n", user_string($arg1), user_string($arg2), $arg3); } +/** + * Triggered upon function return, either global or + * local + * + * @param arg0 pid + * @param arg1 MFA of the returned from function + * @param arg2 depth + */ probe process("beam.smp").mark("function-return") { printf("pid %s return %s depth %d\n", user_string($arg1), user_string($arg2), $arg3); } +/** + * Triggered on built-in function entry + * + * @param arg0 pid + * @param arg1 MFA of the function + */ probe process("beam.smp").mark("bif-entry") { printf("pid %s BIF entry mfa %s\n", user_string($arg1), user_string($arg2)); } +/** + * Triggered on built-in function return + * + * @param arg0 pid + * @param arg1 MFA of the returned from function + */ probe process("beam.smp").mark("bif-return") { printf("pid %s BIF return mfa %s\n", user_string($arg1), user_string($arg2)); } +/** + * Triggered on native function entry + * + * @param arg0 pid + * @param arg1 MFA of the function + */ probe process("beam.smp").mark("nif-entry") { printf("pid %s NIF entry mfa %s\n", user_string($arg1), user_string($arg2)); } +/** + * Triggered upon native function return + * + * @param arg0 pid + * @param arg1 MFA of the returned from function + */ probe process("beam.smp").mark("nif-return") { printf("pid %s NIF return mfa %s\n", user_string($arg1), user_string($arg2)); diff --git a/lib/sasl/Makefile b/lib/sasl/Makefile index 065eb45fbb..06af80fd35 100644 --- a/lib/sasl/Makefile +++ b/lib/sasl/Makefile @@ -36,3 +36,6 @@ SPECIAL_TARGETS = # include $(ERL_TOP)/make/otp_subdir.mk +DIA_PLT_APPS=tools + +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/sasl/src/systools_lib.erl b/lib/sasl/src/systools_lib.erl index dd97aeff08..f5489e7900 100644 --- a/lib/sasl/src/systools_lib.erl +++ b/lib/sasl/src/systools_lib.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -63,8 +63,8 @@ read_term(File) -> end. read_term_from_stream(Stream, File) -> - _ = epp:set_encoding(Stream), - R = io:request(Stream, {get_until,'',erl_scan,tokens,[1]}), + Encoding = epp:set_encoding(Stream), + R = io:request(Stream, {get_until,Encoding,'',erl_scan,tokens,[1]}), case R of {ok,Toks,_EndLine} -> case erl_parse:parse_term(Toks) of diff --git a/lib/snmp/Makefile b/lib/snmp/Makefile index df321fc2d1..52debf1670 100644 --- a/lib/snmp/Makefile +++ b/lib/snmp/Makefile @@ -2,7 +2,7 @@ # %CopyrightBegin% # -# Copyright Ericsson AB 1996-2019. All Rights Reserved. +# Copyright Ericsson AB 1996-2016. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -71,24 +71,6 @@ do_configure: configure configure: configure.in autoconf -.PHONY: info gclean - -info: - @echo "OS: $(OS)" - @echo "DOCB: $(DOCB)" - @echo "" - @echo "SNMP_VSN: $(SNMP_VSN)" - @echo "APP_VSN: $(APP_VSN)" - @echo "" - @echo "DIA_PLT: $(DIA_PLT)" - @echo "DIA_ANALYSIS: $(DIA_ANALYSIS)" - @echo "" - - -gclean: - git clean -fXd - - # ---------------------------------------------------- # Application (source) release targets # ---------------------------------------------------- @@ -130,28 +112,6 @@ tar: $(APP_TAR_FILE) $(APP_TAR_FILE): $(APP_DIR) (cd $(APP_RELEASE_DIR); gtar zcf $(APP_TAR_FILE) $(DIR_NAME)) -dclean: - rm -f $(DIA_PLT) - rm -f $(DIA_ANALYSIS) - -dialyzer_plt: $(DIA_PLT) - -$(DIA_PLT): Makefile - @echo "Building $(APPLICATION) plt file" - @dialyzer --build_plt \ - --output_plt $@ \ - -r ../$(APPLICATION)/ebin \ - ../../lib/kernel/ebin \ - ../../lib/stdlib/ebin \ - ../../lib/runtime_tools/ebin \ - ../../lib/crypto/ebin \ - ../../lib/mnesia/ebin \ - ../../erts/preloaded/ebin \ - --output $(DIA_ANALYSIS) \ - --verbose - -dialyzer: $(DIA_PLT) - @echo "Running dialyzer on $(APPLICATION)" - @dialyzer --plt $< \ - ../$(APPLICATION)/ebin \ - --verbose +DIA_PLT_APPS=runtime_tools crypto mnesia + +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/snmp/src/compile/snmpc_misc.erl b/lib/snmp/src/compile/snmpc_misc.erl index 312074f2e7..d0fee418e1 100644 --- a/lib/snmp/src/compile/snmpc_misc.erl +++ b/lib/snmp/src/compile/snmpc_misc.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2003-2016. All Rights Reserved. +%% Copyright Ericsson AB 2003-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -156,7 +156,8 @@ loop(Fd, Res, Func, StartLine, File) -> %% io:read modified to give us line numbers. %%----------------------------------------------------------------- read(Io, Prompt, StartLine) -> - case io:request(Io, {get_until, Prompt, erl_scan, tokens, [StartLine]}) of + Enc = latin1, + case io:request(Io, {get_until, Enc, Prompt, erl_scan, tokens, [StartLine]}) of {ok, Toks, EndLine} -> case erl_parse:parse_term(Toks) of {ok, Term} -> diff --git a/lib/snmp/src/misc/snmp_conf.erl b/lib/snmp/src/misc/snmp_conf.erl index 20b7af0373..bec9d4d9d9 100644 --- a/lib/snmp/src/misc/snmp_conf.erl +++ b/lib/snmp/src/misc/snmp_conf.erl @@ -265,7 +265,8 @@ open_file(File) -> end. do_read(Io, Prompt, StartLine) -> - case io:request(Io, {get_until,Prompt,erl_scan,tokens,[StartLine]}) of + Enc = latin1, + case io:request(Io, {get_until,Enc,Prompt,erl_scan,tokens,[StartLine]}) of {ok, Toks, EndLine} -> case erl_parse:parse_term(Toks) of {ok, Term} -> diff --git a/lib/snmp/src/misc/snmp_config.erl b/lib/snmp/src/misc/snmp_config.erl index 3104f2a096..5aab9a74e0 100644 --- a/lib/snmp/src/misc/snmp_config.erl +++ b/lib/snmp/src/misc/snmp_config.erl @@ -2737,7 +2737,8 @@ read_lines(Fd, Acc, StartLine) -> end. read_and_parse_term(Fd, StartLine) -> - case io:request(Fd, {get_until, "", erl_scan, tokens, [StartLine]}) of + Enc = latin1, + case io:request(Fd, {get_until, Enc, "", erl_scan, tokens, [StartLine]}) of {ok, Tokens, EndLine} -> case erl_parse:parse_term(Tokens) of {ok, Term} -> diff --git a/lib/ssh/Makefile b/lib/ssh/Makefile index dedc7ac3a6..b96cc2bbaa 100644 --- a/lib/ssh/Makefile +++ b/lib/ssh/Makefile @@ -37,4 +37,6 @@ SPECIAL_TARGETS = # include $(ERL_TOP)/make/otp_subdir.mk +DIA_PLT_APPS=crypto runtime_tools public_key +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/ssh/src/Makefile b/lib/ssh/src/Makefile index 9627b70eeb..46bf6332e5 100644 --- a/lib/ssh/src/Makefile +++ b/lib/ssh/src/Makefile @@ -55,6 +55,7 @@ MODULES= \ ssh_app \ ssh_auth\ ssh_bits \ + ssh_channel_sup \ ssh_cli \ ssh_connection \ ssh_connection_handler \ @@ -66,7 +67,6 @@ MODULES= \ ssh_message \ ssh_no_io \ ssh_options \ - ssh_server_channel_sup \ ssh_sftp \ ssh_sftpd \ ssh_sftpd_file\ @@ -171,7 +171,7 @@ $(EBIN)/ssh_connection_handler.$(EMULATOR): ssh_connection_handler.erl ssh.hrl \ $(EBIN)/ssh_shell.$(EMULATOR): ssh_shell.erl ssh_connect.hrl $(EBIN)/ssh_system_sup.$(EMULATOR): ssh_system_sup.erl ssh.hrl $(EBIN)/ssh_subsystem_sup.$(EMULATOR): ssh_subsystem_sup.erl -$(EBIN)/ssh_server_channel_sup.$(EMULATOR): ssh_server_channel_sup.erl +$(EBIN)/ssh_channel_sup.$(EMULATOR): ssh_channel_sup.erl ssh.hrl $(EBIN)/ssh_acceptor_sup.$(EMULATOR): ssh_acceptor_sup.erl ssh.hrl $(EBIN)/ssh_acceptor.$(EMULATOR): ssh_acceptor.erl ssh.hrl $(EBIN)/ssh_app.$(EMULATOR): ssh_app.erl diff --git a/lib/ssh/src/ssh.app.src b/lib/ssh/src/ssh.app.src index 2193c14611..379cde23e3 100644 --- a/lib/ssh/src/ssh.app.src +++ b/lib/ssh/src/ssh.app.src @@ -11,6 +11,7 @@ ssh_auth, ssh_message, ssh_bits, + ssh_channel_sup, ssh_cli, ssh_client_channel, ssh_client_key_api, @@ -28,7 +29,6 @@ ssh_info, ssh_no_io, ssh_server_channel, - ssh_server_channel_sup, ssh_server_key_api, ssh_sftp, ssh_sftpd, diff --git a/lib/ssh/src/ssh.erl b/lib/ssh/src/ssh.erl index 28f8e0ac18..874c545e61 100644 --- a/lib/ssh/src/ssh.erl +++ b/lib/ssh/src/ssh.erl @@ -128,15 +128,13 @@ connect(Socket, UserOptions, NegotiationTimeout) when is_port(Socket), {error, Error} -> {error, Error}; Options -> - case valid_socket_to_use(Socket, ?GET_OPT(transport,Options)) of - ok -> - {ok, {Host,_Port}} = inet:peername(Socket), - Opts = ?PUT_INTERNAL_OPT([{user_pid,self()}, {host,Host}], Options), - ssh_connection_handler:start_connection(client, Socket, Opts, NegotiationTimeout); - {error,SockError} -> - {error,SockError} - end - end; + case valid_socket_to_use(Socket, ?GET_OPT(transport,Options)) of + ok -> + connect_socket(Socket, Options, NegotiationTimeout); + {error,SockError} -> + {error,SockError} + end + end; connect(Host, Port, Options) when is_integer(Port), Port>0, @@ -150,9 +148,9 @@ connect(Host, Port, Options) when is_integer(Port), Options :: client_options(), NegotiationTimeout :: timeout(). -connect(Host0, Port, UserOptions, Timeout) when is_integer(Port), - Port>0, - is_list(UserOptions) -> +connect(Host0, Port, UserOptions, NegotiationTimeout) when is_integer(Port), + Port>0, + is_list(UserOptions) -> case ssh_options:handle_options(client, UserOptions) of {error, _Reason} = Error -> Error; @@ -163,8 +161,7 @@ connect(Host0, Port, UserOptions, Timeout) when is_integer(Port), Host = mangle_connect_address(Host0, SocketOpts), try Transport:connect(Host, Port, SocketOpts, ConnectionTimeout) of {ok, Socket} -> - Opts = ?PUT_INTERNAL_OPT([{user_pid,self()}, {host,Host}], Options), - ssh_connection_handler:start_connection(client, Socket, Opts, Timeout); + connect_socket(Socket, Options, NegotiationTimeout); {error, Reason} -> {error, Reason} catch @@ -175,6 +172,22 @@ connect(Host0, Port, UserOptions, Timeout) when is_integer(Port), end end. + +connect_socket(Socket, Options0, NegotiationTimeout) -> + {ok, {Host,Port}} = inet:sockname(Socket), + Profile = ?GET_OPT(profile, Options0), + {ok, SystemSup} = sshc_sup:start_child(Host, Port, Profile, Options0), + {ok, SubSysSup} = ssh_system_sup:start_subsystem(SystemSup, client, Host, Port, Profile, Options0), + ConnectionSup = ssh_system_sup:connection_supervisor(SystemSup), + Opts = ?PUT_INTERNAL_OPT([{user_pid,self()}, + {host,Host}, + {supervisors, [{system_sup, SystemSup}, + {subsystem_sup, SubSysSup}, + {connection_sup, ConnectionSup}]} + ], Options0), + ssh_connection_handler:start_connection(client, Socket, Opts, NegotiationTimeout). + + %%-------------------------------------------------------------------- -spec close(ConnectionRef) -> ok | {error,term()} when ConnectionRef :: connection_ref() . @@ -451,7 +464,7 @@ stop_listener(Address, Port, Profile) -> -spec stop_daemon(DaemonRef::daemon_ref()) -> ok. stop_daemon(SysSup) -> - ssh_system_sup:stop_system(SysSup). + ssh_system_sup:stop_system(server, SysSup). -spec stop_daemon(inet:ip_address(), inet:port_number()) -> ok. @@ -464,11 +477,11 @@ stop_daemon(Address, Port) -> stop_daemon(any, Port, Profile) -> map_ip(fun(IP) -> - ssh_system_sup:stop_system(IP, Port, Profile) + ssh_system_sup:stop_system(server, IP, Port, Profile) end, [{0,0,0,0},{0,0,0,0,0,0,0,0}]); stop_daemon(Address, Port, Profile) -> map_ip(fun(IP) -> - ssh_system_sup:stop_system(IP, Port, Profile) + ssh_system_sup:stop_system(server, IP, Port, Profile) end, {address,Address}). %%-------------------------------------------------------------------- diff --git a/lib/ssh/src/ssh_server_channel_sup.erl b/lib/ssh/src/ssh_channel_sup.erl index ff74061bb3..4b36c8a5a0 100644 --- a/lib/ssh/src/ssh_server_channel_sup.erl +++ b/lib/ssh/src/ssh_channel_sup.erl @@ -22,11 +22,12 @@ %%---------------------------------------------------------------------- %% Purpose: Ssh channel supervisor. %%---------------------------------------------------------------------- --module(ssh_server_channel_sup). +-module(ssh_channel_sup). -behaviour(supervisor). +-include("ssh.hrl"). --export([start_link/1, start_child/5]). +-export([start_link/1, start_child/8]). %% Supervisor callback -export([init/1]). @@ -37,15 +38,36 @@ start_link(Args) -> supervisor:start_link(?MODULE, [Args]). -start_child(Sup, Callback, Id, Args, Exec) -> + +start_child(client, ChannelSup, ConnRef, Callback, Id, Args, Exec, _Opts) when is_pid(ConnRef) -> + start_the_child(ssh_client_channel, ChannelSup, ConnRef, Callback, Id, Args, Exec); +start_child(server, ChannelSup, ConnRef, Callback, Id, Args, Exec, Opts) when is_pid(ConnRef) -> + case max_num_channels_not_exceeded(ChannelSup, Opts) of + true -> + start_the_child(ssh_server_channel, ChannelSup, ConnRef, Callback, Id, Args, Exec); + false -> + {error, max_num_channels_exceeded} + end. + + +start_the_child(ChanMod, ChannelSup, ConnRef, Callback, Id, Args, Exec) -> ChildSpec = #{id => make_ref(), - start => {ssh_server_channel, start_link, [self(), Id, Callback, Args, Exec]}, + start => {ChanMod, start_link, [ConnRef, Id, Callback, Args, Exec]}, restart => temporary, type => worker, - modules => [ssh_server_channel] + modules => [ChanMod] }, - supervisor:start_child(Sup, ChildSpec). + case supervisor:start_child(ChannelSup, ChildSpec) of + {ok, Pid} -> + {ok, Pid}; + {ok, Pid, _Info} -> + {ok,Pid}; + {error, {Error,_Info}} -> + {error, Error}; + {error, Error} -> + {error, Error} + end. %%%========================================================================= %%% Supervisor callback @@ -60,3 +82,9 @@ init(_Args) -> %%%========================================================================= %%% Internal functions %%%========================================================================= +max_num_channels_not_exceeded(ChannelSup, Opts) -> + MaxNumChannels = ?GET_OPT(max_channels, Opts), + NumChannels = length([x || {_,_,worker,[ssh_server_channel]} <- + supervisor:which_children(ChannelSup)]), + %% Note that NumChannels is BEFORE starting a new one + NumChannels < MaxNumChannels. diff --git a/lib/ssh/src/ssh_connection.erl b/lib/ssh/src/ssh_connection.erl index c5316bf133..067e5230c9 100644 --- a/lib/ssh/src/ssh_connection.erl +++ b/lib/ssh/src/ssh_connection.erl @@ -648,19 +648,14 @@ handle_msg(#ssh_msg_channel_request{recipient_channel = ChannelId, #channel{remote_id=RemoteId} = Channel = ssh_client_channel:cache_lookup(Cache, ChannelId), Reply = - try - start_subsystem(SsName, Connection, Channel, - {subsystem, ChannelId, WantReply, binary_to_list(SsName)}) - of + case start_subsystem(SsName, Connection, Channel, + {subsystem, ChannelId, WantReply, binary_to_list(SsName)}) of {ok, Pid} -> erlang:monitor(process, Pid), ssh_client_channel:cache_update(Cache, Channel#channel{user=Pid}), channel_success_msg(RemoteId); {error,_Error} -> channel_failure_msg(RemoteId) - catch - _:_ -> - channel_failure_msg(RemoteId) end, {[{connection_reply,Reply}], Connection}; @@ -921,7 +916,7 @@ start_cli(#connection{options = Options, no_cli -> {error, cli_disabled}; {CbModule, Args} -> - start_channel(CbModule, ChannelId, Args, SubSysSup, Exec, Options) + ssh_subsystem_sup:start_channel(server, SubSysSup, self(), CbModule, ChannelId, Args, Exec, Options) end. @@ -931,37 +926,15 @@ start_subsystem(BinName, #connection{options = Options, Name = binary_to_list(BinName), case check_subsystem(Name, Options) of {Callback, Opts} when is_atom(Callback), Callback =/= none -> - start_channel(Callback, ChannelId, Opts, SubSysSup, Options); - {Other, _} when Other =/= none -> + ssh_subsystem_sup:start_channel(server, SubSysSup, self(), Callback, ChannelId, Opts, undefined, Options); + {none, _} -> + {error, bad_subsystem}; + {_, _} -> {error, legacy_option_not_supported} end. %%% Helpers for starting cli/subsystems -start_channel(Cb, Id, Args, SubSysSup, Opts) -> - start_channel(Cb, Id, Args, SubSysSup, undefined, Opts). - -start_channel(Cb, Id, Args, SubSysSup, Exec, Opts) -> - ChannelSup = ssh_subsystem_sup:channel_supervisor(SubSysSup), - case max_num_channels_not_exceeded(ChannelSup, Opts) of - true -> - case ssh_server_channel_sup:start_child(ChannelSup, Cb, Id, Args, Exec) of - {error,{Error,_Info}} -> - throw(Error); - Others -> - Others - end; - false -> - throw(max_num_channels_exceeded) - end. - -max_num_channels_not_exceeded(ChannelSup, Opts) -> - MaxNumChannels = ?GET_OPT(max_channels, Opts), - NumChannels = length([x || {_,_,worker,[ssh_server_channel]} <- - supervisor:which_children(ChannelSup)]), - %% Note that NumChannels is BEFORE starting a new one - NumChannels < MaxNumChannels. - check_subsystem("sftp"= SsName, Options) -> case ?GET_OPT(subsystems, Options) of no_subsys -> % FIXME: Can 'no_subsys' ever be matched? @@ -1300,13 +1273,13 @@ handle_cli_msg(C0, ChId, Reply0) -> Ch0 = ssh_client_channel:cache_lookup(Cache, ChId), case Ch0#channel.user of undefined -> - case (catch start_cli(C0, ChId)) of + case start_cli(C0, ChId) of {ok, Pid} -> erlang:monitor(process, Pid), Ch = Ch0#channel{user = Pid}, ssh_client_channel:cache_update(Cache, Ch), reply_msg(Ch, C0, Reply0); - _Other -> + {error, _Error} -> Reply = {connection_reply, channel_failure_msg(Ch0#channel.remote_id)}, {[Reply], C0} end; diff --git a/lib/ssh/src/ssh_connection_handler.erl b/lib/ssh/src/ssh_connection_handler.erl index ad5a54a2e2..69d3c3b18f 100644 --- a/lib/ssh/src/ssh_connection_handler.erl +++ b/lib/ssh/src/ssh_connection_handler.erl @@ -48,6 +48,7 @@ -export([start_connection/4, available_hkey_algorithms/2, open_channel/6, + start_channel/5, request/6, request/7, reply_request/3, send/5, @@ -126,35 +127,29 @@ stop(ConnectionHandler)-> timeout() ) -> {ok, connection_ref()} | {error, term()}. %% . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . -start_connection(client = Role, Socket, Options, Timeout) -> +start_connection(Role, Socket, Options, Timeout) -> try - {ok, Pid} = sshc_sup:start_child([Role, Socket, Options]), - ok = socket_control(Socket, Pid, Options), - handshake(Pid, erlang:monitor(process,Pid), Timeout) - catch - exit:{noproc, _} -> - {error, ssh_not_started}; - _:Error -> - {error, Error} - end; - -start_connection(server = Role, Socket, Options, Timeout) -> - try - case ?GET_OPT(parallel_login, Options) of - true -> - HandshakerPid = - spawn_link(fun() -> - receive - {do_handshake, Pid} -> - handshake(Pid, erlang:monitor(process,Pid), Timeout) - end - end), - ChildPid = start_the_connection_child(HandshakerPid, Role, Socket, Options), - HandshakerPid ! {do_handshake, ChildPid}; - false -> - ChildPid = start_the_connection_child(self(), Role, Socket, Options), - handshake(ChildPid, erlang:monitor(process,ChildPid), Timeout) - end + case Role of + client -> + ChildPid = start_the_connection_child(self(), Role, Socket, Options), + handshake(ChildPid, erlang:monitor(process,ChildPid), Timeout); + server -> + case ?GET_OPT(parallel_login, Options) of + true -> + HandshakerPid = + spawn_link(fun() -> + receive + {do_handshake, Pid} -> + handshake(Pid, erlang:monitor(process,Pid), Timeout) + end + end), + ChildPid = start_the_connection_child(HandshakerPid, Role, Socket, Options), + HandshakerPid ! {do_handshake, ChildPid}; + false -> + ChildPid = start_the_connection_child(self(), Role, Socket, Options), + handshake(ChildPid, erlang:monitor(process,ChildPid), Timeout) + end + end catch exit:{noproc, _} -> {error, ssh_not_started}; @@ -176,6 +171,8 @@ disconnect(Code, DetailedText, Module, Line) -> [{next_event, internal, {send_disconnect, Code, DetailedText, Module, Line}}]}). %%-------------------------------------------------------------------- +%%% Open a channel in the connection to the peer, that is, do the ssh +%%% signalling with the peer. -spec open_channel(connection_ref(), string(), iodata(), @@ -195,6 +192,18 @@ open_channel(ConnectionHandler, Timeout}). %%-------------------------------------------------------------------- +%%% Start a channel handling process in the superviser tree +-spec start_channel(connection_ref(), atom(), channel_id(), list(), term()) -> + {ok, pid()} | {error, term()}. + +%% . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . +start_channel(ConnectionHandler, CallbackModule, ChannelId, Args, Exec) -> + {ok, {SubSysSup,Role,Opts}} = call(ConnectionHandler, get_misc), + ssh_subsystem_sup:start_channel(Role, SubSysSup, + ConnectionHandler, CallbackModule, ChannelId, + Args, Exec, Opts). + +%%-------------------------------------------------------------------- -spec request(connection_ref(), pid(), channel_id(), @@ -381,8 +390,21 @@ alg(ConnectionHandler) -> %% . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . init_connection_handler(Role, Socket, Opts) -> case init([Role, Socket, Opts]) of - {ok, StartState, D} -> + {ok, StartState, D} when Role == server -> + process_flag(trap_exit, true), + gen_statem:enter_loop(?MODULE, + [], %%[{debug,[trace,log,statistics,debug]} ], %% [] + StartState, + D); + + {ok, StartState, D0=#data{connection_state=C}} when Role == client -> process_flag(trap_exit, true), + Sups = ?GET_INTERNAL_OPT(supervisors, Opts), + D = D0#data{connection_state = + C#connection{system_supervisor = proplists:get_value(system_sup, Sups), + sub_system_supervisor = proplists:get_value(subsystem_sup, Sups), + connection_supervisor = proplists:get_value(connection_sup, Sups) + }}, gen_statem:enter_loop(?MODULE, [], %%[{debug,[trace,log,statistics,debug]} ], %% [] StartState, @@ -1250,6 +1272,13 @@ handle_event({call,From}, {eof, ChannelId}, StateName, D0) {keep_state, D0, [{reply,From,{error,closed}}]} end; +handle_event({call,From}, get_misc, StateName, + #data{connection_state = #connection{options = Opts}} = D) when ?CONNECTED(StateName) -> + Sups = ?GET_INTERNAL_OPT(supervisors, Opts), + SubSysSup = proplists:get_value(subsystem_sup, Sups), + Reply = {ok, {SubSysSup, role(StateName), Opts}}, + {keep_state, D, [{reply,From,Reply}]}; + handle_event({call,From}, {open, ChannelPid, Type, InitialWindowSize, MaxPacketSize, Data, Timeout}, StateName, @@ -1664,17 +1693,36 @@ start_the_connection_child(UserPid, Role, Socket, Options0) -> Sups = ?GET_INTERNAL_OPT(supervisors, Options0), ConnectionSup = proplists:get_value(connection_sup, Sups), Options = ?PUT_INTERNAL_OPT({user_pid,UserPid}, Options0), - {ok, Pid} = ssh_connection_sup:start_child(ConnectionSup, [Role, Socket, Options]), - ok = socket_control(Socket, Pid, Options), + InitArgs = [Role, Socket, Options], + {ok, Pid} = ssh_connection_sup:start_child(ConnectionSup, InitArgs), + ok = socket_control(Socket, Pid, Options), % transfer the Socket ownership in a controlled way. Pid. %%-------------------------------------------------------------------- %% Stopping -stop_subsystem(#data{connection_state = +stop_subsystem(#data{ssh_params = + #ssh{role = Role}, + connection_state = #connection{system_supervisor = SysSup, - sub_system_supervisor = SubSysSup}}) when is_pid(SubSysSup) -> - ssh_system_sup:stop_subsystem(SysSup, SubSysSup); + sub_system_supervisor = SubSysSup} + }) when is_pid(SysSup) andalso is_pid(SubSysSup) -> + process_flag(trap_exit, false), + C = self(), + spawn(fun() -> + Mref = erlang:monitor(process, C), + receive + {'DOWN', Mref, process, C, _Info} -> ok + after + 10000 -> ok + end, + case Role of + client -> + ssh_system_sup:stop_system(Role, SysSup); + _ -> + ssh_system_sup:stop_subsystem(SysSup, SubSysSup) + end + end); stop_subsystem(_) -> ok. @@ -1766,6 +1814,8 @@ call(FsmPid, Event, Timeout) -> exit:{normal, _R} -> {error, closed}; exit:{{shutdown, _R},_} -> + {error, closed}; + exit:{shutdown, _R} -> {error, closed} end. diff --git a/lib/ssh/src/ssh_info.erl b/lib/ssh/src/ssh_info.erl index 79cd95e422..91365205aa 100644 --- a/lib/ssh/src/ssh_info.erl +++ b/lib/ssh/src/ssh_info.erl @@ -79,8 +79,8 @@ print_clients() -> lists:map(fun print_client/1, supervisor:which_children(sshc_sup)) catch - C:E -> - io_lib:format('***print_clients FAILED: ~p:~p~n',[C,E]) + C:E:S -> + io_lib:format('***print_clients FAILED: ~p:~p,~n ~p~n',[C,E,S]) end. print_client({undefined,Pid,supervisor,[ssh_connection_handler]}) -> @@ -94,9 +94,9 @@ print_client({undefined,Pid,supervisor,[ssh_connection_handler]}) -> io_lib:format(?INDENT?INDENT?INDENT"No channels~n",[]) end]; -print_client(Other) -> - io_lib:format(" [[Other 1: ~p]]~n",[Other]). - +print_client({{client,ssh_system_sup,_,_,_},Pid,supervisor,[ssh_system_sup]}) when is_pid(Pid) -> + lists:map(fun print_system_sup/1, + supervisor:which_children(Pid)). %%%================================================================ print_servers() -> @@ -104,8 +104,8 @@ print_servers() -> lists:map(fun print_server/1, supervisor:which_children(sshd_sup)) catch - C:E -> - io_lib:format('***print_servers FAILED: ~p:~p~n',[C,E]) + C:E:S -> + io_lib:format('***print_servers FAILED: ~p:~p,~n ~p~n',[C,E,S]) end. @@ -140,22 +140,33 @@ print_system_sup({{ssh_acceptor_sup,_LocalHost,_LocalPort,_Profile}, Pid, superv -print_channels({{server,ssh_server_channel_sup,_,_},Pid,supervisor,[ssh_server_channel_sup]}) when is_pid(Pid) -> +print_channels({{Role,ssh_channel_sup,_,_},Pid,supervisor,[ssh_channel_sup]}) when is_pid(Pid) -> + ChanBehaviour = + case Role of + server -> ssh_server_channel; + client -> ssh_client_channel + end, Children = supervisor:which_children(Pid), - ChannelPids = [P || {R,P,worker,[ssh_server_channel]} <- Children, + ChannelPids = [P || {R,P,worker,[Mod]} <- Children, + ChanBehaviour == Mod, is_pid(P), is_reference(R)], case ChannelPids of [] -> io_lib:format(?INDENT?INDENT"No channels~n",[]); [Ch1Pid|_] -> - {{ConnManager,_}, _Str} = ssh_server_channel:get_print_info(Ch1Pid), + {{ConnManager,_}, _Str} = ChanBehaviour:get_print_info(Ch1Pid), {{_,Remote},_} = ssh_connection_handler:get_print_info(ConnManager), [io_lib:format(?INDENT?INDENT"Remote: ~s ConnectionRef = ~p~n",[fmt_host_port(Remote),ConnManager]), lists:map(fun print_ch/1, ChannelPids) ] end; -print_channels({{server,ssh_connection_sup,_,_},Pid,supervisor,[ssh_connection_sup]}) when is_pid(Pid) -> - []. % The supervisor of the connections socket owning process +print_channels({{_Role,ssh_connection_sup,_,_},Pid,supervisor,[ssh_connection_sup]}) when is_pid(Pid) -> + []; % The supervisor of the connections socket owning process + +print_channels({Ref,Pid,supervisor,[ssh_tcpip_forward_acceptor_sup]}) when is_pid(Pid), + is_reference(Ref) -> + []. % The supervisor of the forward_acceptor process + print_ch(Pid) -> try diff --git a/lib/ssh/src/ssh_sftp.erl b/lib/ssh/src/ssh_sftp.erl index 4b6e187c3a..305c1c31fd 100644 --- a/lib/ssh/src/ssh_sftp.erl +++ b/lib/ssh/src/ssh_sftp.erl @@ -162,8 +162,8 @@ start_channel(Cm, UserOptions) when is_pid(Cm) -> {_SshOpts, ChanOpts, SftpOpts} = handle_options(UserOptions), case ssh_xfer:attach(Cm, [], ChanOpts) of {ok, ChannelId, Cm} -> - case ssh_client_channel:start(Cm, ChannelId, - ?MODULE, [Cm, ChannelId, SftpOpts]) of + case ssh_connection_handler:start_channel(Cm, ?MODULE, ChannelId, + [Cm,ChannelId,SftpOpts], undefined) of {ok, Pid} -> case wait_for_version_negotiation(Pid, Timeout) of ok -> @@ -172,9 +172,7 @@ start_channel(Cm, UserOptions) when is_pid(Cm) -> TimeOut end; {error, Reason} -> - {error, format_channel_start_error(Reason)}; - ignore -> - {error, ignore} + {error, format_channel_start_error(Reason)} end; Error -> Error @@ -200,7 +198,8 @@ start_channel(Host, Port, UserOptions) -> proplists:get_value(timeout, SftpOpts, infinity)), case ssh_xfer:connect(Host, Port, SshOpts, ChanOpts, Timeout) of {ok, ChannelId, Cm} -> - case ssh_client_channel:start(Cm, ChannelId, ?MODULE, [Cm,ChannelId,SftpOpts]) of + case ssh_connection_handler:start_channel(Cm, ?MODULE, ChannelId, + [Cm,ChannelId,SftpOpts], undefined) of {ok, Pid} -> case wait_for_version_negotiation(Pid, Timeout) of ok -> @@ -209,9 +208,7 @@ start_channel(Host, Port, UserOptions) -> TimeOut end; {error, Reason} -> - {error, format_channel_start_error(Reason)}; - ignore -> - {error, ignore} + {error, format_channel_start_error(Reason)} end; Error -> Error diff --git a/lib/ssh/src/ssh_subsystem_sup.erl b/lib/ssh/src/ssh_subsystem_sup.erl index 5fc8f7e764..db34735e1b 100644 --- a/lib/ssh/src/ssh_subsystem_sup.erl +++ b/lib/ssh/src/ssh_subsystem_sup.erl @@ -30,7 +30,8 @@ -export([start_link/5, connection_supervisor/1, - channel_supervisor/1 + channel_supervisor/1, + start_channel/8 ]). %% Supervisor callback @@ -46,9 +47,13 @@ connection_supervisor(SupPid) -> Children = supervisor:which_children(SupPid), ssh_connection_sup(Children). -channel_supervisor(SupPid) -> +channel_supervisor(SupPid) when is_pid(SupPid) -> Children = supervisor:which_children(SupPid), - ssh_server_channel_sup(Children). + ssh_channel_sup(Children). + +start_channel(Role, SupPid, ConnRef, Callback, Id, Args, Exec, Opts) -> + ChannelSup = channel_supervisor(SupPid), + ssh_channel_sup:start_child(Role, ChannelSup, ConnRef, Callback, Id, Args, Exec, Opts). %%%========================================================================= %%% Supervisor callback @@ -64,11 +69,9 @@ init([Role, Address, Port, Profile, Options]) -> %%%========================================================================= %%% Internal functions %%%========================================================================= -child_specs(client, _Address, _Port, _Profile, _Options) -> - []; -child_specs(server, Address, Port, Profile, Options) -> - [ssh_channel_child_spec(server, Address, Port, Profile, Options), - ssh_connection_child_spec(server, Address, Port, Profile, Options)]. +child_specs(Role, Address, Port, Profile, Options) -> + [ssh_channel_child_spec(Role, Address, Port, Profile, Options), + ssh_connection_child_spec(Role, Address, Port, Profile, Options)]. ssh_connection_child_spec(Role, Address, Port, _Profile, Options) -> #{id => id(Role, ssh_connection_sup, Address, Port), @@ -78,8 +81,8 @@ ssh_connection_child_spec(Role, Address, Port, _Profile, Options) -> }. ssh_channel_child_spec(Role, Address, Port, _Profile, Options) -> - #{id => id(Role, ssh_server_channel_sup, Address, Port), - start => {ssh_server_channel_sup, start_link, [Options]}, + #{id => id(Role, ssh_channel_sup, Address, Port), + start => {ssh_channel_sup, start_link, [Options]}, restart => temporary, type => supervisor }. @@ -92,10 +95,10 @@ ssh_connection_sup([{_, Child, _, [ssh_connection_sup]} | _]) -> ssh_connection_sup([_ | Rest]) -> ssh_connection_sup(Rest). -ssh_server_channel_sup([{_, Child, _, [ssh_server_channel_sup]} | _]) -> +ssh_channel_sup([{_, Child, _, [ssh_channel_sup]} | _]) -> Child; -ssh_server_channel_sup([_ | Rest]) -> - ssh_server_channel_sup(Rest). +ssh_channel_sup([_ | Rest]) -> + ssh_channel_sup(Rest). diff --git a/lib/ssh/src/ssh_system_sup.erl b/lib/ssh/src/ssh_system_sup.erl index 80406a6c47..704c17c4e7 100644 --- a/lib/ssh/src/ssh_system_sup.erl +++ b/lib/ssh/src/ssh_system_sup.erl @@ -31,9 +31,9 @@ -include("ssh.hrl"). --export([start_link/4, stop_listener/1, - stop_listener/3, stop_system/1, - stop_system/3, system_supervisor/3, +-export([start_link/5, stop_listener/1, + stop_listener/3, stop_system/2, + stop_system/4, system_supervisor/3, subsystem_supervisor/1, channel_supervisor/1, connection_supervisor/1, acceptor_supervisor/1, start_subsystem/6, @@ -50,14 +50,14 @@ %%%========================================================================= %%% API %%%========================================================================= -start_link(Address, Port, Profile, Options) -> +start_link(Role, Address, Port, Profile, Options) -> Name = make_name(Address, Port, Profile), - supervisor:start_link({local, Name}, ?MODULE, [Address, Port, Profile, Options]). + supervisor:start_link({local, Name}, ?MODULE, [Role, Address, Port, Profile, Options]). %%%========================================================================= %%% Supervisor callback %%%========================================================================= -init([Address, Port, Profile, Options]) -> +init([server, Address, Port, Profile, Options]) -> SupFlags = #{strategy => one_for_one, intensity => 0, period => 3600 @@ -73,6 +73,14 @@ init([Address, Port, Profile, Options]) -> _ -> [] end, + {ok, {SupFlags,ChildSpecs}}; + +init([client, _Address, _Port, _Profile, _Options]) -> + SupFlags = #{strategy => one_for_one, + intensity => 0, + period => 3600 + }, + ChildSpecs = [], {ok, {SupFlags,ChildSpecs}}. %%%========================================================================= @@ -92,11 +100,10 @@ stop_listener(Address, Port, Profile) -> system_supervisor(Address, Port, Profile)). -stop_system(SysSup) -> - catch sshd_sup:stop_child(SysSup), - ok. +stop_system(server, SysSup) -> catch sshd_sup:stop_child(SysSup), ok; +stop_system(client, SysSup) -> catch sshc_sup:stop_child(SysSup), ok. -stop_system(Address, Port, Profile) -> +stop_system(server, Address, Port, Profile) -> catch sshd_sup:stop_child(Address, Port, Profile), ok. diff --git a/lib/ssh/src/sshc_sup.erl b/lib/ssh/src/sshc_sup.erl index 869de244ac..0c9f8be5a9 100644 --- a/lib/ssh/src/sshc_sup.erl +++ b/lib/ssh/src/sshc_sup.erl @@ -27,7 +27,10 @@ -behaviour(supervisor). --export([start_link/0, start_child/1]). +-export([start_link/0, + start_child/4, + stop_child/1 + ]). %% Supervisor callback -export([init/1]). @@ -38,22 +41,45 @@ %%% API %%%========================================================================= start_link() -> - supervisor:start_link({local,?SSHC_SUP}, ?MODULE, []). + supervisor:start_link({local,?MODULE}, ?MODULE, []). -start_child(Args) -> - supervisor:start_child(?MODULE, Args). +start_child(Address, Port, Profile, Options) -> + %% Here we a new connction on a new Host/EFERMERAL Port/Profile + Spec = child_spec(Address, Port, Profile, Options), + supervisor:start_child(?MODULE, Spec). + +stop_child(ChildId) when is_tuple(ChildId) -> + supervisor:terminate_child(?SSHC_SUP, ChildId); +stop_child(ChildPid) when is_pid(ChildPid)-> + stop_child(system_name(ChildPid)). %%%========================================================================= %%% Supervisor callback %%%========================================================================= init(_) -> - SupFlags = #{strategy => simple_one_for_one, + SupFlags = #{strategy => one_for_one, intensity => 0, period => 3600 }, - ChildSpecs = [#{id => undefined, % As simple_one_for_one is used. - start => {ssh_connection_handler, start_link, []}, - restart => temporary % because there is no way to restart a crashed connection - } - ], + ChildSpecs = [], {ok, {SupFlags,ChildSpecs}}. + +%%%========================================================================= +%%% Internal functions +%%%========================================================================= +child_spec(Address, Port, Profile, Options) -> + #{id => id(Address, Port, Profile), + start => {ssh_system_sup, start_link, [client, Address, Port, Profile, Options]}, + restart => temporary, + type => supervisor + }. + +id(Address, Port, Profile) -> + {client, ssh_system_sup, Address, Port, Profile}. + +system_name(SysSup) -> + case lists:keyfind(SysSup, 2, supervisor:which_children(?SSHC_SUP)) of + {Name, SysSup, _, _} -> Name; + false -> undefind + end. + diff --git a/lib/ssh/src/sshd_sup.erl b/lib/ssh/src/sshd_sup.erl index b5361abba5..b716b66ec7 100644 --- a/lib/ssh/src/sshd_sup.erl +++ b/lib/ssh/src/sshd_sup.erl @@ -89,7 +89,7 @@ init(_) -> %%%========================================================================= child_spec(Address, Port, Profile, Options) -> #{id => id(Address, Port, Profile), - start => {ssh_system_sup, start_link, [Address, Port, Profile, Options]}, + start => {ssh_system_sup, start_link, [server, Address, Port, Profile, Options]}, restart => temporary, type => supervisor }. diff --git a/lib/ssh/test/ssh_sup_SUITE.erl b/lib/ssh/test/ssh_sup_SUITE.erl index a0e3d809be..0bf4d87f0a 100644 --- a/lib/ssh/test/ssh_sup_SUITE.erl +++ b/lib/ssh/test/ssh_sup_SUITE.erl @@ -116,18 +116,21 @@ sshc_subtree(Config) when is_list(Config) -> {user_interaction, false}, {user, ?USER}, {password, ?PASSWD},{user_dir, UserDir}]), - ?wait_match([{_, _,worker,[ssh_connection_handler]}], - supervisor:which_children(sshc_sup)), + ?wait_match([{{client,ssh_system_sup, LocalIP, LocalPort, ?DEFAULT_PROFILE}, + SysSup, supervisor,[ssh_system_sup]}], + supervisor:which_children(sshc_sup), + [SysSup, LocalIP, LocalPort]), + check_sshc_system_tree(SysSup, Pid1, LocalIP, LocalPort, Config), {ok, Pid2} = ssh:connect(Host, Port, [{silently_accept_hosts, true}, {user_interaction, false}, {user, ?USER}, {password, ?PASSWD}, {user_dir, UserDir}]), - ?wait_match([{_,_,worker,[ssh_connection_handler]}, - {_,_,worker,[ssh_connection_handler]}], + ?wait_match([{_, _,supervisor,[ssh_system_sup]}, + {_, _,supervisor,[ssh_system_sup]}], supervisor:which_children(sshc_sup)), ssh:close(Pid1), - ?wait_match([{_,_,worker,[ssh_connection_handler]}], + ?wait_match([{_, _,supervisor,[ssh_system_sup]}], supervisor:which_children(sshc_sup)), ssh:close(Pid2), ?wait_match([], supervisor:which_children(sshc_sup)). @@ -217,6 +220,7 @@ killed_acceptor_restarts(Config) -> ct:log("~s",[lists:flatten(ssh_info:string())]), %% Make acceptor restart: + ct:pal("Expect a SUPERVISOR REPORT with offender {pid,~p}....~n", [AccPid]), exit(AccPid, kill), ?wait_match(undefined, process_info(AccPid)), @@ -226,6 +230,8 @@ killed_acceptor_restarts(Config) -> AccPid1, 500, 30), + ct:pal("... now there should not be any SUPERVISOR REPORT.~n", []), + true = (AccPid1 =/= AccPid2), %% Connect second client and check it is alive: @@ -339,9 +345,9 @@ chk_empty_con_daemon(Daemon) -> ?wait_match([{{server,ssh_connection_sup, _,_}, ConnectionSup, supervisor, [ssh_connection_sup]}, - {{server,ssh_server_channel_sup,_ ,_}, + {{server,ssh_channel_sup,_ ,_}, ChannelSup,supervisor, - [ssh_server_channel_sup]}], + [ssh_channel_sup]}], supervisor:which_children(SubSysSup), [ConnectionSup,ChannelSup]), ?wait_match([{{ssh_acceptor_sup,_,_,_},_,worker,[ssh_acceptor]}], @@ -372,9 +378,9 @@ check_sshd_system_tree(Daemon, Config) -> ?wait_match([{{server,ssh_connection_sup, _,_}, ConnectionSup, supervisor, [ssh_connection_sup]}, - {{server,ssh_server_channel_sup,_ ,_}, + {{server,ssh_channel_sup,_ ,_}, ChannelSup,supervisor, - [ssh_server_channel_sup]}], + [ssh_channel_sup]}], supervisor:which_children(SubSysSup), [ConnectionSup,ChannelSup]), @@ -386,12 +392,55 @@ check_sshd_system_tree(Daemon, Config) -> ?wait_match([], supervisor:which_children(ChannelSup)), - ssh_sftp:start_channel(Client), + {ok,PidC} = ssh_sftp:start_channel(Client), + + ?wait_match([{_, PidS,worker,[ssh_server_channel]}], + supervisor:which_children(ChannelSup), + [PidS]), + true = (PidS =/= PidC), - ?wait_match([{_, _,worker,[ssh_server_channel]}], - supervisor:which_children(ChannelSup)), ssh:close(Client). + +check_sshc_system_tree(SysSup, Connection, LocalIP, LocalPort, _Config) -> + ?wait_match([{_,SubSysSup,supervisor,[ssh_subsystem_sup]}], + supervisor:which_children(SysSup), + [SubSysSup]), + ?wait_match([{{client,ssh_connection_sup, LocalIP, LocalPort}, + ConnectionSup, supervisor, + [ssh_connection_sup]}, + {{client,ssh_channel_sup, LocalIP, LocalPort}, + ChannelSup,supervisor, + [ssh_channel_sup]}], + supervisor:which_children(SubSysSup), + [ConnectionSup,ChannelSup]), + ?wait_match([{_, Connection, worker,[ssh_connection_handler]}], + supervisor:which_children(ConnectionSup)), + ?wait_match([], supervisor:which_children(ChannelSup)), + + {ok,ChPid1} = ssh_sftp:start_channel(Connection), + ?wait_match([{_,ChPid1,worker,[ssh_client_channel]}], + supervisor:which_children(ChannelSup)), + + {ok,ChPid2} = ssh_sftp:start_channel(Connection), + ?wait_match([{_,ChPidA,worker,[ssh_client_channel]}, + {_,ChPidB,worker,[ssh_client_channel]}], + supervisor:which_children(ChannelSup), + [ChPidA, ChPidB]), + lists:sort([ChPidA, ChPidB]) == lists:sort([ChPid1, ChPid2]), + + ct:pal("Expect a SUPERVISOR REPORT with offender {pid,~p}....~n", [ChPid1]), + exit(ChPid1, kill), + ?wait_match([{_,ChPid2,worker,[ssh_client_channel]}], + supervisor:which_children(ChannelSup)), + + ct:pal("Expect a SUPERVISOR REPORT with offender {pid,~p}....~n", [ChPid2]), + exit(ChPid2, kill), + ?wait_match([], supervisor:which_children(ChannelSup)), + ct:pal("... now there should not be any SUPERVISOR REPORT.~n", []). + + + acceptor_pid(DaemonPid) -> Parent = self(), Pid = spawn(fun() -> diff --git a/lib/ssl/Makefile b/lib/ssl/Makefile index bd43794a36..526560288f 100644 --- a/lib/ssl/Makefile +++ b/lib/ssl/Makefile @@ -38,4 +38,6 @@ SPECIAL_TARGETS = # include $(ERL_TOP)/make/otp_subdir.mk +DIA_PLT_APPS=crypto runtime_tools inets public_key +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl index 1c8a2ca452..a407694617 100644 --- a/lib/ssl/src/ssl_handshake.erl +++ b/lib/ssl/src/ssl_handshake.erl @@ -724,7 +724,7 @@ encode_extensions([#psk_key_exchange_modes{ke_modes = KEModes0} | Rest], Acc) -> encode_extensions([#pre_shared_key_client_hello{ offered_psks = #offered_psks{ identities = Identities0, - binders = Binders0} = PSKs} | Rest], Acc) -> + binders = Binders0}} | Rest], Acc) -> Identities = encode_psk_identities(Identities0), Binders = encode_psk_binders(Binders0), Len = byte_size(Identities) + byte_size(Binders), diff --git a/lib/stdlib/Makefile b/lib/stdlib/Makefile index 3086d85445..cae3844126 100644 --- a/lib/stdlib/Makefile +++ b/lib/stdlib/Makefile @@ -35,3 +35,7 @@ SPECIAL_TARGETS = # Default Subdir Targets # include $(ERL_TOP)/make/otp_subdir.mk + +DIA_PLT_APPS=compiler crypto + +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/stdlib/doc/src/erl_parse.xml b/lib/stdlib/doc/src/erl_parse.xml index 8142e5c0aa..d487cccdfc 100644 --- a/lib/stdlib/doc/src/erl_parse.xml +++ b/lib/stdlib/doc/src/erl_parse.xml @@ -69,6 +69,25 @@ <name name="erl_parse_tree"></name> </datatype> <datatype> + <name>af_binelement(_)</name> + <desc> + <p>Abstract representation of an element of a bitstring.</p> + </desc> + </datatype> + <datatype> + <name>af_generator()</name> + <desc> + <p>Abstract representation of a generator + or a bitstring generator.</p> + </desc> + </datatype> + <datatype> + <name>af_remote_function()></name> + <desc> + <p>Abstract representation of a remote function call.</p> + </desc> + </datatype> + <datatype> <name name="error_description"></name> </datatype> <datatype> diff --git a/lib/stdlib/doc/src/io_protocol.xml b/lib/stdlib/doc/src/io_protocol.xml index 84b5f62c7f..f05c358866 100644 --- a/lib/stdlib/doc/src/io_protocol.xml +++ b/lib/stdlib/doc/src/io_protocol.xml @@ -5,7 +5,7 @@ <header> <copyright> <year>1999</year> - <year>2016</year> + <year>2019</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -168,16 +168,6 @@ ok returns it "as is".</item> </list> - <p>For backward compatibility, the following <c>Request</c>s are also to be - handled by an I/O server (they are not to be present after - Erlang/OTP R15B):</p> - - <pre> -{put_chars, Characters} -{put_chars, Module, Function, Args}</pre> - - <p>These are to behave as <c>{put_chars, latin1, Characters}</c> and - <c>{put_chars, latin1, Module, Function, Args}</c>, respectively.</p> </section> <section> @@ -332,19 +322,6 @@ eof </item> </list> - <p>For backward compatibility, the following <c>Request</c>s are also to be - handled by an I/O server (they are not to be present after - Erlang/OTP R15B):</p> - - <pre> -{get_until, Prompt, Module, Function, ExtraArgs} -{get_chars, Prompt, N} -{get_line, Prompt}</pre> - - <p>These are to behave as - <c>{get_until, latin1, Prompt, Module, Function, ExtraArgs}</c>, - <c>{get_chars, latin1, Prompt, N}</c>, and - <c>{get_line, latin1, Prompt}</c>, respectively.</p> </section> <section> @@ -637,24 +614,6 @@ request({requests, Reqs}, State) -> function applying the requests in the list one after another, returning the last result.</p> - <p>We need to handle backward compatibility and the - <seealso marker="kernel:file"><c>file</c></seealso> module (which - uses the old requests until backward compatibility with pre-R13 nodes is - no longer needed). Notice that the I/O server does not work with a simple - <c>file:write/2</c> if these are not added:</p> - - <code> -request({put_chars,Chars}, State) -> - request({put_chars,latin1,Chars}, State); -request({put_chars,M,F,As}, State) -> - request({put_chars,latin1,M,F,As}, State); -request({get_chars,Prompt,N}, State) -> - request({get_chars,latin1,Prompt,N}, State); -request({get_line,Prompt}, State) -> - request({get_line,latin1,Prompt}, State); -request({get_until, Prompt,M,F,As}, State) -> - request({get_until,latin1,Prompt,M,F,As}, State);</code> - <p><c>{error, request}</c> must be returned if the request is not recognized:</p> diff --git a/lib/stdlib/src/edlin.erl b/lib/stdlib/src/edlin.erl index f027d05f55..6078c5e67b 100644 --- a/lib/stdlib/src/edlin.erl +++ b/lib/stdlib/src/edlin.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2018. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -352,9 +352,6 @@ do_op({blink,C,M}, Bef=[$$,$$|_], Aft, Rs) -> %% don't blink after a $ do_op({blink,C,_}, Bef=[$$|_], Aft, Rs) -> do_op({insert,C}, Bef, Aft, Rs); -%do_op({blink,C,M}, Bef, [], Rs) -> -% N = over_paren(Bef, C, M), -% {blink,N+1,{[C|Bef],[]},[{move_rel,-(N+1)},{put_chars,[C]}|Rs]}; do_op({blink,C,M}, Bef, Aft, Rs) -> case over_paren(Bef, C, M) of beep -> diff --git a/lib/stdlib/src/erl_internal.erl b/lib/stdlib/src/erl_internal.erl index 939abaff00..e1f8fe151e 100644 --- a/lib/stdlib/src/erl_internal.erl +++ b/lib/stdlib/src/erl_internal.erl @@ -245,11 +245,14 @@ bif(M, F, A) when is_atom(M), is_atom(F), is_integer(A) -> false. bif(abs, 1) -> true; bif(apply, 2) -> true; bif(apply, 3) -> true; +bif(atom_to_binary, 1) -> true; bif(atom_to_binary, 2) -> true; bif(atom_to_list, 1) -> true; bif(binary_part, 2) -> true; bif(binary_part, 3) -> true; +bif(binary_to_atom, 1) -> true; bif(binary_to_atom, 2) -> true; +bif(binary_to_existing_atom, 1) -> true; bif(binary_to_existing_atom, 2) -> true; bif(binary_to_integer, 1) -> true; bif(binary_to_integer, 2) -> true; diff --git a/lib/stdlib/src/erl_parse.yrl b/lib/stdlib/src/erl_parse.yrl index 4ad94f2507..ca53f992f6 100644 --- a/lib/stdlib/src/erl_parse.yrl +++ b/lib/stdlib/src/erl_parse.yrl @@ -604,6 +604,8 @@ Erlang code. -export_type([abstract_clause/0, abstract_expr/0, abstract_form/0, abstract_type/0, form_info/0, error_info/0]). +%% The following types are exported because they are used by syntax_tools +-export_type([af_binelement/1, af_generator/0, af_remote_function/0]). %% Start of Abstract Format diff --git a/lib/stdlib/src/io.erl b/lib/stdlib/src/io.erl index 63c9a6bddf..1848aa3628 100644 --- a/lib/stdlib/src/io.erl +++ b/lib/stdlib/src/io.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2018. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -106,7 +106,6 @@ nl() -> IoDevice :: device(). nl(Io) -> -% o_request(Io, {put_chars,io_lib:nl()}). o_request(Io, nl, nl). -spec columns() -> {'ok', pos_integer()} | {'error', 'enotsup'}. @@ -255,8 +254,6 @@ read(Io, Prompt) -> case request(Io, {get_until,unicode,Prompt,erl_scan,tokens,[1]}) of {ok,Toks,_EndLine} -> erl_parse:parse_term(Toks); -% {error, Reason} when atom(Reason) -> -% erlang:error(conv_reason(read, Reason), [Io, Prompt]); {error,E,_EndLine} -> {error,E}; {eof,_EndLine} -> @@ -352,12 +349,7 @@ fread(Prompt, Format) -> | server_no_data(). fread(Io, Prompt, Format) -> - case request(Io, {fread,Prompt,Format}) of -% {error, Reason} when atom(Reason) -> -% erlang:error(conv_reason(fread, Reason), [Io, Prompt, Format]); - Other -> - Other - end. + request(Io, {fread,Prompt,Format}). -spec format(Format) -> 'ok' when Format :: format(). diff --git a/lib/stdlib/src/io_lib.erl b/lib/stdlib/src/io_lib.erl index 21d66c5529..e2823b70f2 100644 --- a/lib/stdlib/src/io_lib.erl +++ b/lib/stdlib/src/io_lib.erl @@ -78,7 +78,7 @@ %% Utilities for collecting characters. -export([collect_chars/3, collect_chars/4, - collect_line/2, collect_line/3, collect_line/4, + collect_line/3, collect_line/4, get_until/3, get_until/4]). %% The following functions were used by Yecc's include-file. @@ -851,6 +851,7 @@ collect_chars({binary,Stack,N}, Data,latin1, _) -> end; collect_chars({list,Stack,N}, Data, _,_) -> collect_chars_list(Stack, N, Data); + %% collect_chars(Continuation, MoreChars, Count) %% Returns: %% {done,Result,RestChars} @@ -881,32 +882,6 @@ collect_chars_list(Stack, N, []) -> collect_chars_list(Stack,N, [H|T]) -> collect_chars_list([H|Stack], N-1, T). -%% collect_line(Continuation, MoreChars) -%% Returns: -%% {done,Result,RestChars} -%% {more,Continuation} -%% -%% XXX Can be removed when compatibility with pre-R12B-5 nodes -%% is no longer required. -%% -collect_line([], Chars) -> - collect_line1(Chars, []); -collect_line({SoFar}, More) -> - collect_line1(More, SoFar). - -collect_line1([$\r, $\n|Rest], Stack) -> - collect_line1([$\n|Rest], Stack); -collect_line1([$\n|Rest], Stack) -> - {done,lists:reverse([$\n|Stack], []),Rest}; -collect_line1([C|Rest], Stack) -> - collect_line1(Rest, [C|Stack]); -collect_line1(eof, []) -> - {done,eof,[]}; -collect_line1(eof, Stack) -> - {done,lists:reverse(Stack, []),[]}; -collect_line1([], Stack) -> - {more,{Stack}}. - %% collect_line(State, Data, _). New in R9C. %% Returns: %% {stop,Result,RestData} diff --git a/lib/stdlib/src/io_lib_pretty.erl b/lib/stdlib/src/io_lib_pretty.erl index 77f02eafe0..838d412d0c 100644 --- a/lib/stdlib/src/io_lib_pretty.erl +++ b/lib/stdlib/src/io_lib_pretty.erl @@ -895,9 +895,6 @@ write_string(S, _Uni) -> io_lib:write_string(S, $"). %" expand({_, _, _Dots=0, no_more} = If, _T, _Dd) -> If; -%% expand({{list,L}, _Len, _, no_more}, T, Dd) -> -%% {NL, NLen, NDots} = expand_list(L, T, Dd, 2), -%% {{list,NL}, NLen, NDots, no_more}; expand({{tuple,IsTagged,L}, _Len, _, no_more}, T, Dd) -> {NL, NLen, NDots} = expand_list(L, T, Dd, 2), {{tuple,IsTagged,NL}, NLen, NDots, no_more}; diff --git a/lib/stdlib/src/stdlib.app.src b/lib/stdlib/src/stdlib.app.src index 5b7d2218c7..4c4fb4349e 100644 --- a/lib/stdlib/src/stdlib.app.src +++ b/lib/stdlib/src/stdlib.app.src @@ -111,4 +111,3 @@ {runtime_dependencies, ["sasl-3.0","kernel-6.0","erts-10.5","crypto-3.3", "compiler-5.0"]} ]}. - diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index 4fcc0e65ea..b1813052d2 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -47,7 +47,8 @@ test_delete_table_while_size_snapshot/1, test_delete_table_while_size_snapshot_helper/0]). -export([ordered/1, ordered_match/1, interface_equality/1, - fixtable_next/1, fixtable_insert/1, rename/1, rename_unnamed/1, evil_rename/1, + fixtable_next/1, fixtable_iter_bag/1, + fixtable_insert/1, rename/1, rename_unnamed/1, evil_rename/1, update_element/1, update_counter/1, evil_update_counter/1, partly_bound/1, match_heavy/1]). -export([update_counter_with_default/1]). -export([update_counter_table_growth/1]). @@ -128,7 +129,7 @@ all() -> {group, match}, t_match_spec_run, {group, lookup_element}, {group, misc}, {group, files}, {group, heavy}, ordered, ordered_match, - interface_equality, fixtable_next, fixtable_insert, + interface_equality, fixtable_next, fixtable_iter_bag, fixtable_insert, rename, rename_unnamed, evil_rename, update_element, update_counter, evil_update_counter, update_counter_with_default, partly_bound, @@ -2450,6 +2451,135 @@ do_fixtable_next(Tab) -> false = ets:info(Tab, fixed), ets:delete(Tab). +%% Check that iteration of bags find all live objects and nothing else. +fixtable_iter_bag(Config) when is_list(Config) -> + repeat_for_opts(fun fixtable_iter_do/1, + [write_concurrency,[bag,duplicate_bag]]). + +fixtable_iter_do(Opts) -> + EtsMem = etsmem(), + do_fixtable_iter_bag(ets_new(fixtable_iter_bag,Opts)), + verify_etsmem(EtsMem). + +do_fixtable_iter_bag(T) -> + MaxValues = 4, + %% Create 1 to MaxValues objects for each key + %% and then delete every possible combination of those objects + %% in every possible order. + %% Then test iteration returns all live objects and nothing else. + + CrDelOps = [begin + Values = lists:seq(1,N), + %% All ways of deleting any number of the Values in any order + Combos = combs(Values), + DeleteOps = concat_lists([perms(C) || C <- Combos]), + {N, DeleteOps} + end + || N <- lists:seq(1,MaxValues)], + + %%io:format("~p\n", [CrDelOps]), + + NKeys = lists:foldl(fun({_, DeleteOps}, Cnt) -> + Cnt + length(DeleteOps) + end, + 0, + CrDelOps), + + io:format("Create ~p keys\n", [NKeys]), + + %% Fixate even before inserts just to maintain small table size + %% and increase likelyhood of different keys in same bucket. + ets:safe_fixtable(T,true), + InsRes = [begin + [begin + Key = {NValues,ValueList}, + [begin + Tpl = {Key, V}, + %%io:format("Insert object ~p", [Tpl]), + ets:insert(T, Tpl), + Tpl + end + || V <- lists:seq(1,NValues)] + end + || ValueList <- DeleteOps] + end + || {NValues, DeleteOps} <- CrDelOps], + + Inserted = lists:flatten(InsRes), + InSorted = lists:sort(Inserted), + InSorted = lists:usort(Inserted), %% No duplicates + NObjs = length(Inserted), + + DelRes = [begin + [begin + Key = {NValues,ValueList}, + [begin + Tpl = {Key, V}, + %%io:format("Delete object ~p", [Tpl]), + ets:delete_object(T, Tpl), + Tpl + end + || V <- ValueList] + end + || ValueList <- DeleteOps] + end + || {NValues, DeleteOps} <- CrDelOps], + + Deleted = lists:flatten(DelRes), + DelSorted = lists:sort(Deleted), + DelSorted = lists:usort(Deleted), %% No duplicates + NDels = length(Deleted), + + %% Nr of keys where all values were deleted. + NDeletedKeys = lists:sum([factorial(N) || N <- lists:seq(1,MaxValues)]), + + CountKeysFun = fun Me(K1, Cnt) -> + case ets:next(T, K1) of + '$end_of_table' -> + Cnt; + K2 -> + Objs = ets:lookup(T, K2), + [{{NValues, ValueList}, _V} | _] = Objs, + ExpectedLive = NValues - length(ValueList), + ExpectedLive = length(Objs), + Me(K2, Cnt+1) + end + end, + + ExpectedKeys = NKeys - NDeletedKeys, + io:format("Expected keys: ~p\n", [ExpectedKeys]), + FoundKeys = CountKeysFun(ets:first(T), 1), + io:format("Found keys: ~p\n", [FoundKeys]), + ExpectedKeys = FoundKeys, + + ExpectedObjs = NObjs - NDels, + io:format("Expected objects: ~p\n", [ExpectedObjs]), + FoundObjs = ets:select_count(T, [{{'_','_'}, [], [true]}]), + io:format("Found objects: ~p\n", [FoundObjs]), + ExpectedObjs = FoundObjs, + + ets:delete(T). + +%% All permutations of list +perms([]) -> [[]]; +perms(L) -> [[H|T] || H <- L, T <- perms(L--[H])]. + +%% All combinations of picking the element (or not) from list +combs([]) -> [[]]; +combs([H|T]) -> + Tcombs = combs(T), + Tcombs ++ [[H | C] || C <- Tcombs]. + +factorial(0) -> 1; +factorial(N) when N > 0 -> + N * factorial(N - 1). + +concat_lists([]) -> + []; +concat_lists([H|T]) -> + H ++ concat_lists(T). + + %% Check inserts of deleted keys in fixed bags. fixtable_insert(Config) when is_list(Config) -> Combos = [[Type,{write_concurrency,WC}] || Type<- [bag,duplicate_bag], @@ -4971,7 +5101,7 @@ tabfile_ext4(Config) when is_list(Config) -> {error,Y} = ets:file2tab(FName,[{verify,true}]), ets:tab2file(TL,FName,[{extended_info,[md5sum]}]), {X,Y} - end || N <- lists:seq(500,600)], + end || N <- lists:seq(700,800)], io:format("~p~n",[Res]), file:delete(FName) end), diff --git a/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html b/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html index 27d6849c60..239877c257 100644 --- a/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html +++ b/lib/stdlib/test/ets_SUITE_data/visualize_throughput.html @@ -4,7 +4,7 @@ <!-- %% --> <!-- %% %CopyrightBegin% --> <!-- %% --> -<!-- %% Copyright Ericsson AB and Kjell Winblad 1996-2018. All Rights Reserved. --> +<!-- %% Copyright Ericsson AB and Kjell Winblad 1996-2019. All Rights Reserved. --> <!-- %% --> <!-- %% Licensed under the Apache License, Version 2.0 (the "License"); --> <!-- %% you may not use this file except in compliance with the License. --> @@ -44,6 +44,12 @@ <br> <textarea id="dataField" rows="4" cols="50">#bench_data_placeholder</textarea> <br> + <input type="checkbox" id="throughputPlot" checked> Include Throughput Plot + <br> + <input type="checkbox" id="betterThanWorstPlot"> Include % More Throughput Than Worst Plot + <br> + <input type="checkbox" id="worseThanBestPlot"> Include % Less Throughput Than Best Plot + <br> <input type="checkbox" id="barPlot"> Bar Plot <br> <input type="checkbox" id="sameSpacing" checked> Same X Spacing Between Points @@ -148,10 +154,52 @@ } return data; } + function toCompareData(dataParam, compareWithWorst) { + var data = $.extend(true, [], dataParam); + var worstSoFarMap = {}; + var defaultSoFarValue = compareWithWorst ? Number.MAX_VALUE : Number.MIN_VALUE; + function getWorstBestSoFar(x){ + return worstSoFarMap[x] === undefined ? defaultSoFarValue : worstSoFarMap[x]; + } + function setWorstBestSoFar(x, y){ + return worstSoFarMap[x] = y; + } + function lessOrGreaterThan(n1, n2){ + return compareWithWorst ? n1 < n2 : n1 > n2; + } + $.each(data, function(i, allResConfig) { + $.each(allResConfig.y, function(index, res) { + var xName = allResConfig.x[index]; + if(lessOrGreaterThan(res, getWorstBestSoFar(xName))){ + setWorstBestSoFar(xName, res); + } + }); + }); + $.each(data, function(i, allResConfig) { + $.each(allResConfig.y, function(index, res) { + var xName = allResConfig.x[index]; + if(compareWithWorst){ + allResConfig.y[index] = ((res / getWorstBestSoFar(xName))-1.0) * 100; + }else{ + allResConfig.y[index] = (1.0 -(res / getWorstBestSoFar(xName))) * 100; + } + }); + }); + return data; + } + function toBetterThanWorstData(data){ + return toCompareData(data, true); + } + function toWorseThanBestData(data){ + return toCompareData(data, false); + } function plotGraphs(){ var insertPlaceholder = $("#insertPlaceholder"); var sameSpacing = $('#sameSpacing').is(":checked"); var barPlot = $('#barPlot').is(":checked"); + var throughputPlot = $('#throughputPlot').is(":checked"); + var betterThanWorstPlot = $('#betterThanWorstPlot').is(":checked"); + var worseThanBestPlot = $('#worseThanBestPlot').is(":checked"); var lines = $("#dataField").val(); $('.showCheck').each(function() { var item = $(this); @@ -188,42 +236,59 @@ plotGraph(lines, sameSpacing, barPlot, prefix)); } } + var nrOfGraphs = 0; + function plotScenario(name, plotType) { + var data = scenarioDataMap[name]; + var yAxisTitle = undefined; + nrOfGraphs = nrOfGraphs + 1; + $("<div class='added' id='graph" + nrOfGraphs + "'>") + .insertBefore(insertPlaceholder); + $("<button type='button' class='added' id='fullscreenButton" + nrOfGraphs + "'>Fill screen</button>") + .insertBefore(insertPlaceholder); + $("<span class='added'><br><hr><br></span>") + .insertBefore(insertPlaceholder); + if (plotType === 'throughput') { + yAxisTitle = 'Operations/Second'; + } else if (plotType === 'better_than_worst') { + yAxisTitle = '% More Throughput Than Worst'; + data = toBetterThanWorstData(data); + } else { + yAxisTitle = '% Less Throughput Than Best'; + data = toWorseThanBestData(data); + } + var layout = { + title: name, + xaxis: { + title: '# of Processes' + }, + yaxis: { + title: yAxisTitle + } + }; + $("#fullscreenButton" + nrOfGraphs).click( + function () { + $('#graph' + nrOfGraphs).replaceWith( + $("<div class='added' id='graph" + nrOfGraphs + "'>")); + layout = $.extend({}, layout, { + width: $(window).width() - 40, + height: $(window).height() - 40 + }); + Plotly.newPlot('graph' + nrOfGraphs, data, layout); + }); + Plotly.newPlot('graph' + nrOfGraphs, data, layout); + } $.each(scenarioList, - function( index, name ) { - var nrOfGraphs = index + 1; - var data = scenarioDataMap[name]; - $( "<div class='added' id='graph"+nrOfGraphs+"'>") - .insertBefore( insertPlaceholder ); - $( "<button type='button' class='added' id='fullscreenButton"+nrOfGraphs+"'>Fill screen</button>") - .insertBefore( insertPlaceholder ); - $( "<span class='added'><br><hr><br></span>") - .insertBefore( insertPlaceholder ); - var layout = { - title:name, - xaxis: { - title: '# of Processes' - }, - yaxis: { - title: 'Operations/Second' - } - - }; - - $("#fullscreenButton"+nrOfGraphs).click( - function(){ - $('#graph'+nrOfGraphs).replaceWith( - $("<div class='added' id='graph"+nrOfGraphs+"'>")); - layout = $.extend({}, layout, { - width:$(window).width()-40, - height:$(window).height()-40 - }); - Plotly.newPlot('graph'+nrOfGraphs, data, layout); - }); - Plotly.newPlot('graph'+nrOfGraphs, data, layout); - - }); - - + function (index, name) { + if (throughputPlot) { + plotScenario(name, 'throughput'); + } + if (betterThanWorstPlot) { + plotScenario(name, 'better_than_worst'); + } + if (worseThanBestPlot) { + plotScenario(name, 'worse_than_best'); + } + }); } $(document).ready(function(){ $('#renderButton').click( diff --git a/lib/stdlib/test/io_proto_SUITE.erl b/lib/stdlib/test/io_proto_SUITE.erl index e497b2fb5d..df6958cfa9 100644 --- a/lib/stdlib/test/io_proto_SUITE.erl +++ b/lib/stdlib/test/io_proto_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2017. All Rights Reserved. +%% Copyright Ericsson AB 2009-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1568,10 +1568,6 @@ request({put_chars, Encoding, Chars}, State) -> request({put_chars, Encoding, Module, Function, Args}, State) -> {ok, ok, State#state{q=[{put_chars, Encoding, Module, Function, Args} | State#state.q ]}}; -request({put_chars,Chars}, State) -> - {ok, ok, State#state{q=[{put_chars, Chars} | State#state.q ]}}; -request({put_chars,M,F,As}, State) -> - {ok, ok, State#state{q=[{put_chars, M,F,As} | State#state.q ]}}; request({get_until, Encoding, Prompt, M, F, As}, State) -> {ok, convert(State#state.nxt, Encoding, State#state.mode), State#state{nxt = eof, q = [{get_until, Encoding, Prompt, M, F, As} | State#state.q]}}; request({get_chars, Encoding, Prompt, N}, State) -> @@ -1583,20 +1579,6 @@ request({get_line, Encoding, Prompt}, State) -> State#state{nxt = eof, q = [{get_line, Encoding, Prompt} | State#state.q]}}; -request({get_until, Prompt, M, F, As}, State) -> - {ok, convert(State#state.nxt, latin1, State#state.mode), - State#state{nxt = eof, - q = [{get_until, Prompt, M, F, As} | State#state.q]}}; -request({get_chars, Prompt, N}, State) -> - {ok, convert(State#state.nxt, latin1, State#state.mode), - State#state{nxt = eof, - q = [{get_chars, Prompt, N} | - State#state.q]}}; -request({get_line, Prompt}, State) -> - {ok, convert(State#state.nxt, latin1, State#state.mode), - State#state{nxt = eof, - q = [{get_line, Prompt} | - State#state.q]}}; request({get_geomentry,_}, State) -> {error, {error,enotsup}, State}; request({setopts, Opts}, State) when Opts =:= [{binary, false}]; Opts =:= [list] -> diff --git a/lib/stdlib/test/shell_SUITE.erl b/lib/stdlib/test/shell_SUITE.erl index cdb6031b07..4d85e1f04b 100644 --- a/lib/stdlib/test/shell_SUITE.erl +++ b/lib/stdlib/test/shell_SUITE.erl @@ -3141,25 +3141,16 @@ io_request({get_geometry,columns}, S) -> {ok,80,S}; io_request({get_geometry,rows}, S) -> {ok,24,S}; -io_request({put_chars,Chars}, S) -> - {ok,ok,S#state{reply = [S#state.reply | Chars]}}; io_request({put_chars,latin1,Chars}, S) -> {ok,ok,S#state{reply = [S#state.reply | Chars]}}; io_request({put_chars,unicode,Chars0}, S) -> Chars = unicode:characters_to_list(Chars0), {ok,ok,S#state{reply = [S#state.reply | Chars]}}; -io_request({put_chars,Mod,Func,Args}, S) -> - case catch apply(Mod, Func, Args) of - Chars when is_list(Chars) -> - io_request({put_chars,Chars}, S) - end; io_request({put_chars,Enc,Mod,Func,Args}, S) -> case catch apply(Mod, Func, Args) of Chars when is_list(Chars) -> io_request({put_chars,Enc,Chars}, S) end; -io_request({get_until,_Prompt,Mod,Func,ExtraArgs}, S) -> - get_until(Mod, Func, ExtraArgs, S, latin1); io_request({get_until,Enc,_Prompt,Mod,Func,ExtraArgs}, S) -> get_until(Mod, Func, ExtraArgs, S, Enc). diff --git a/lib/stdlib/test/unicode_util_SUITE_data/GraphemeBreakTest.txt b/lib/stdlib/test/unicode_util_SUITE_data/GraphemeBreakTest.txt index 6847953c23..fb4fec9fff 100644 --- a/lib/stdlib/test/unicode_util_SUITE_data/GraphemeBreakTest.txt +++ b/lib/stdlib/test/unicode_util_SUITE_data/GraphemeBreakTest.txt @@ -1,6 +1,6 @@ -# GraphemeBreakTest-11.0.0.txt -# Date: 2018-03-18, 13:30:33 GMT -# © 2018 Unicode®, Inc. +# GraphemeBreakTest-12.1.0.txt +# Date: 2019-03-10, 10:53:12 GMT +# © 2019 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use, see http://www.unicode.org/terms_of_use.html # @@ -56,8 +56,6 @@ ÷ 0020 × 0308 × 200D ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ 0020 ÷ 0378 ÷ # ÷ [0.2] SPACE (Other) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ 0020 × 0308 ÷ 0378 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 0020 ÷ D800 ÷ # ÷ [0.2] SPACE (Other) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 0020 × 0308 ÷ D800 ÷ # ÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ 000D ÷ 0020 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] SPACE (Other) ÷ [0.3] ÷ 000D ÷ 0308 ÷ 0020 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 000D ÷ 000D ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] @@ -92,8 +90,6 @@ ÷ 000D ÷ 0308 × 200D ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ 000D ÷ 0378 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] <reserved-0378> (Other) ÷ [0.3] ÷ 000D ÷ 0308 ÷ 0378 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 000D ÷ D800 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 000D ÷ 0308 ÷ D800 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ 000A ÷ 0020 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] SPACE (Other) ÷ [0.3] ÷ 000A ÷ 0308 ÷ 0020 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 000A ÷ 000D ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] @@ -128,8 +124,6 @@ ÷ 000A ÷ 0308 × 200D ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ 000A ÷ 0378 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] <reserved-0378> (Other) ÷ [0.3] ÷ 000A ÷ 0308 ÷ 0378 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 000A ÷ D800 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 000A ÷ 0308 ÷ D800 ÷ # ÷ [0.2] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ 0001 ÷ 0020 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] SPACE (Other) ÷ [0.3] ÷ 0001 ÷ 0308 ÷ 0020 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 0001 ÷ 000D ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] @@ -164,8 +158,6 @@ ÷ 0001 ÷ 0308 × 200D ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ 0001 ÷ 0378 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] <reserved-0378> (Other) ÷ [0.3] ÷ 0001 ÷ 0308 ÷ 0378 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 0001 ÷ D800 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 0001 ÷ 0308 ÷ D800 ÷ # ÷ [0.2] <START OF HEADING> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ 034F ÷ 0020 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 034F × 0308 ÷ 0020 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 034F ÷ 000D ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] @@ -200,8 +192,6 @@ ÷ 034F × 0308 × 200D ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ 034F ÷ 0378 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ 034F × 0308 ÷ 0378 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 034F ÷ D800 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 034F × 0308 ÷ D800 ÷ # ÷ [0.2] COMBINING GRAPHEME JOINER (Extend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ 1F1E6 ÷ 0020 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 1F1E6 × 0308 ÷ 0020 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 1F1E6 ÷ 000D ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] @@ -236,8 +226,6 @@ ÷ 1F1E6 × 0308 × 200D ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ 1F1E6 ÷ 0378 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ 1F1E6 × 0308 ÷ 0378 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 1F1E6 ÷ D800 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 1F1E6 × 0308 ÷ D800 ÷ # ÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (RI) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ 0600 × 0020 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] SPACE (Other) ÷ [0.3] ÷ 0600 × 0308 ÷ 0020 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 0600 ÷ 000D ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] @@ -272,8 +260,6 @@ ÷ 0600 × 0308 × 200D ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ 0600 × 0378 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.2] <reserved-0378> (Other) ÷ [0.3] ÷ 0600 × 0308 ÷ 0378 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 0600 ÷ D800 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 0600 × 0308 ÷ D800 ÷ # ÷ [0.2] ARABIC NUMBER SIGN (Prepend) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ 0903 ÷ 0020 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 0903 × 0308 ÷ 0020 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 0903 ÷ 000D ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] @@ -308,8 +294,6 @@ ÷ 0903 × 0308 × 200D ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ 0903 ÷ 0378 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ 0903 × 0308 ÷ 0378 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 0903 ÷ D800 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 0903 × 0308 ÷ D800 ÷ # ÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ 1100 ÷ 0020 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 1100 × 0308 ÷ 0020 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 1100 ÷ 000D ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] @@ -344,8 +328,6 @@ ÷ 1100 × 0308 × 200D ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ 1100 ÷ 0378 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ 1100 × 0308 ÷ 0378 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 1100 ÷ D800 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 1100 × 0308 ÷ D800 ÷ # ÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ 1160 ÷ 0020 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 1160 × 0308 ÷ 0020 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 1160 ÷ 000D ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] @@ -380,8 +362,6 @@ ÷ 1160 × 0308 × 200D ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ 1160 ÷ 0378 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ 1160 × 0308 ÷ 0378 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 1160 ÷ D800 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 1160 × 0308 ÷ D800 ÷ # ÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ 11A8 ÷ 0020 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 11A8 × 0308 ÷ 0020 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 11A8 ÷ 000D ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] @@ -416,8 +396,6 @@ ÷ 11A8 × 0308 × 200D ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ 11A8 ÷ 0378 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ 11A8 × 0308 ÷ 0378 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 11A8 ÷ D800 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 11A8 × 0308 ÷ D800 ÷ # ÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ AC00 ÷ 0020 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ AC00 × 0308 ÷ 0020 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ AC00 ÷ 000D ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] @@ -452,8 +430,6 @@ ÷ AC00 × 0308 × 200D ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ AC00 ÷ 0378 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ AC00 × 0308 ÷ 0378 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ AC00 ÷ D800 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ AC00 × 0308 ÷ D800 ÷ # ÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ AC01 ÷ 0020 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ AC01 × 0308 ÷ 0020 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ AC01 ÷ 000D ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] @@ -488,8 +464,6 @@ ÷ AC01 × 0308 × 200D ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ AC01 ÷ 0378 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ AC01 × 0308 ÷ 0378 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ AC01 ÷ D800 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ AC01 × 0308 ÷ D800 ÷ # ÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ 231A ÷ 0020 ÷ # ÷ [0.2] WATCH (ExtPict) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 231A × 0308 ÷ 0020 ÷ # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 231A ÷ 000D ÷ # ÷ [0.2] WATCH (ExtPict) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] @@ -524,8 +498,6 @@ ÷ 231A × 0308 × 200D ÷ # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ 231A ÷ 0378 ÷ # ÷ [0.2] WATCH (ExtPict) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ 231A × 0308 ÷ 0378 ÷ # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 231A ÷ D800 ÷ # ÷ [0.2] WATCH (ExtPict) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 231A × 0308 ÷ D800 ÷ # ÷ [0.2] WATCH (ExtPict) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ 0300 ÷ 0020 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 0300 × 0308 ÷ 0020 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 0300 ÷ 000D ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] @@ -560,8 +532,6 @@ ÷ 0300 × 0308 × 200D ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ 0300 ÷ 0378 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ 0300 × 0308 ÷ 0378 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 0300 ÷ D800 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 0300 × 0308 ÷ D800 ÷ # ÷ [0.2] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ 200D ÷ 0020 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 200D × 0308 ÷ 0020 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 200D ÷ 000D ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] @@ -596,8 +566,6 @@ ÷ 200D × 0308 × 200D ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ 200D ÷ 0378 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ 200D × 0308 ÷ 0378 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 200D ÷ D800 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 200D × 0308 ÷ D800 ÷ # ÷ [0.2] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ 0378 ÷ 0020 ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 0378 × 0308 ÷ 0020 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] ÷ 0378 ÷ 000D ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] @@ -632,44 +600,6 @@ ÷ 0378 × 0308 × 200D ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] ÷ 0378 ÷ 0378 ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] ÷ 0378 × 0308 ÷ 0378 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ 0378 ÷ D800 ÷ # ÷ [0.2] <reserved-0378> (Other) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ 0378 × 0308 ÷ D800 ÷ # ÷ [0.2] <reserved-0378> (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] -÷ D800 ÷ 0020 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] SPACE (Other) ÷ [0.3] -÷ D800 ÷ 0308 ÷ 0020 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] SPACE (Other) ÷ [0.3] -÷ D800 ÷ 000D ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ D800 ÷ 0308 ÷ 000D ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <CARRIAGE RETURN (CR)> (CR) ÷ [0.3] -÷ D800 ÷ 000A ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ D800 ÷ 0308 ÷ 000A ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [0.3] -÷ D800 ÷ 0001 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] <START OF HEADING> (Control) ÷ [0.3] -÷ D800 ÷ 0308 ÷ 0001 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <START OF HEADING> (Control) ÷ [0.3] -÷ D800 ÷ 034F ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] -÷ D800 ÷ 0308 × 034F ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAPHEME JOINER (Extend) ÷ [0.3] -÷ D800 ÷ 1F1E6 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ D800 ÷ 0308 ÷ 1F1E6 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (RI) ÷ [0.3] -÷ D800 ÷ 0600 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ D800 ÷ 0308 ÷ 0600 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] ARABIC NUMBER SIGN (Prepend) ÷ [0.3] -÷ D800 ÷ 0903 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ D800 ÷ 0308 × 0903 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3] -÷ D800 ÷ 1100 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ D800 ÷ 0308 ÷ 1100 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3] -÷ D800 ÷ 1160 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ D800 ÷ 0308 ÷ 1160 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3] -÷ D800 ÷ 11A8 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ D800 ÷ 0308 ÷ 11A8 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3] -÷ D800 ÷ AC00 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ D800 ÷ 0308 ÷ AC00 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3] -÷ D800 ÷ AC01 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ D800 ÷ 0308 ÷ AC01 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3] -÷ D800 ÷ 231A ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] WATCH (ExtPict) ÷ [0.3] -÷ D800 ÷ 0308 ÷ 231A ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] WATCH (ExtPict) ÷ [0.3] -÷ D800 ÷ 0300 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] -÷ D800 ÷ 0308 × 0300 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] COMBINING GRAVE ACCENT (Extend_ExtCccZwj) ÷ [0.3] -÷ D800 ÷ 200D ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] -÷ D800 ÷ 0308 × 200D ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [0.3] -÷ D800 ÷ 0378 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] <reserved-0378> (Other) ÷ [0.3] -÷ D800 ÷ 0308 ÷ 0378 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [999.0] <reserved-0378> (Other) ÷ [0.3] -÷ D800 ÷ D800 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] <surrogate-D800> (Control) ÷ [0.3] -÷ D800 ÷ 0308 ÷ D800 ÷ # ÷ [0.2] <surrogate-D800> (Control) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [5.0] <surrogate-D800> (Control) ÷ [0.3] ÷ 000D × 000A ÷ 0061 ÷ 000A ÷ 0308 ÷ # ÷ [0.2] <CARRIAGE RETURN (CR)> (CR) × [3.0] <LINE FEED (LF)> (LF) ÷ [4.0] LATIN SMALL LETTER A (Other) ÷ [5.0] <LINE FEED (LF)> (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [0.3] ÷ 0061 × 0308 ÷ # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] COMBINING DIAERESIS (Extend_ExtCccZwj) ÷ [0.3] ÷ 0020 × 200D ÷ 0646 ÷ # ÷ [0.2] SPACE (Other) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] ARABIC LETTER NOON (Other) ÷ [0.3] @@ -695,6 +625,6 @@ ÷ 2701 × 200D × 2701 ÷ # ÷ [0.2] UPPER BLADE SCISSORS (Other) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) × [11.0] UPPER BLADE SCISSORS (Other) ÷ [0.3] ÷ 0061 × 200D ÷ 2701 ÷ # ÷ [0.2] LATIN SMALL LETTER A (Other) × [9.0] ZERO WIDTH JOINER (ZWJ_ExtCccZwj) ÷ [999.0] UPPER BLADE SCISSORS (Other) ÷ [0.3] # -# Lines: 672 +# Lines: 602 # # EOF diff --git a/lib/stdlib/test/unicode_util_SUITE_data/LineBreakTest.txt b/lib/stdlib/test/unicode_util_SUITE_data/LineBreakTest.txt index 0e9e678a85..eb056990a0 100644 --- a/lib/stdlib/test/unicode_util_SUITE_data/LineBreakTest.txt +++ b/lib/stdlib/test/unicode_util_SUITE_data/LineBreakTest.txt @@ -1,6 +1,6 @@ -# LineBreakTest-11.0.0.txt -# Date: 2018-05-20, 09:03:09 GMT -# © 2018 Unicode®, Inc. +# LineBreakTest-12.1.0.txt +# Date: 2019-03-10, 10:53:14 GMT +# © 2019 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use, see http://www.unicode.org/terms_of_use.html # diff --git a/lib/stdlib/test/unicode_util_SUITE_data/NormalizationTest.txt b/lib/stdlib/test/unicode_util_SUITE_data/NormalizationTest.txt index 72a31bcdf1..54f43bdf4d 100644 --- a/lib/stdlib/test/unicode_util_SUITE_data/NormalizationTest.txt +++ b/lib/stdlib/test/unicode_util_SUITE_data/NormalizationTest.txt @@ -1,6 +1,6 @@ -# NormalizationTest-11.0.0.txt -# Date: 2018-02-19, 18:33:08 GMT -# © 2018 Unicode®, Inc. +# NormalizationTest-12.1.0.txt +# Date: 2019-04-01, 09:10:28 GMT +# © 2019 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use, see http://www.unicode.org/terms_of_use.html # @@ -2149,6 +2149,7 @@ 32FC;32FC;32FC;30F0;30F0; # (㋼; ㋼; ㋼; ヰ; ヰ; ) CIRCLED KATAKANA WI 32FD;32FD;32FD;30F1;30F1; # (㋽; ㋽; ㋽; ヱ; ヱ; ) CIRCLED KATAKANA WE 32FE;32FE;32FE;30F2;30F2; # (㋾; ㋾; ㋾; ヲ; ヲ; ) CIRCLED KATAKANA WO +32FF;32FF;32FF;4EE4 548C;4EE4 548C; # (㋿; ㋿; ㋿; 令和; 令和; ) SQUARE ERA NAME REIWA 3300;3300;3300;30A2 30D1 30FC 30C8;30A2 30CF 309A 30FC 30C8; # (㌀; ㌀; ㌀; アパート; アハ◌゚ート; ) SQUARE APAATO 3301;3301;3301;30A2 30EB 30D5 30A1;30A2 30EB 30D5 30A1; # (㌁; ㌁; ㌁; アルファ; アルファ; ) SQUARE ARUHUA 3302;3302;3302;30A2 30F3 30DA 30A2;30A2 30F3 30D8 309A 30A2; # (㌂; ㌂; ㌂; アンペア; アンヘ◌゚ア; ) SQUARE ANPEA @@ -16363,6 +16364,7 @@ FFEE;FFEE;FFEE;25CB;25CB; # (￮; ￮; ￮; ○; ○; ) HALFWIDTH WHITE CIRCLE 1F14F;1F14F;1F14F;0057 0043;0057 0043; # (🅏; 🅏; 🅏; WC; WC; ) SQUARED WC 1F16A;1F16A;1F16A;004D 0043;004D 0043; # (🅪; 🅪; 🅪; MC; MC; ) RAISED MC SIGN 1F16B;1F16B;1F16B;004D 0044;004D 0044; # (🅫; 🅫; 🅫; MD; MD; ) RAISED MD SIGN +1F16C;1F16C;1F16C;004D 0052;004D 0052; # (🅬; 🅬; 🅬; MR; MR; ) RAISED MR SIGN 1F190;1F190;1F190;0044 004A;0044 004A; # (🆐; 🆐; 🆐; DJ; DJ; ) SQUARE DJ 1F200;1F200;1F200;307B 304B;307B 304B; # (🈀; 🈀; 🈀; ほか; ほか; ) SQUARE HIRAGANA HOKA 1F201;1F201;1F201;30B3 30B3;30B3 30B3; # (🈁; 🈁; 🈁; ココ; ココ; ) SQUARED KATAKANA KOKO @@ -17685,6 +17687,8 @@ FFEE;FFEE;FFEE;25CB;25CB; # (￮; ￮; ￮; ○; ○; ) HALFWIDTH WHITE CIRCLE 0061 0EB8 0EC8 0EB8 0E48 0062;0061 0E48 0EB8 0EB8 0EC8 0062;0061 0E48 0EB8 0EB8 0EC8 0062;0061 0E48 0EB8 0EB8 0EC8 0062;0061 0E48 0EB8 0EB8 0EC8 0062; # (a◌ຸ◌່◌ຸ◌่b; a◌่◌ຸ◌ຸ◌່b; a◌่◌ຸ◌ຸ◌່b; a◌่◌ຸ◌ຸ◌່b; a◌่◌ຸ◌ຸ◌່b; ) LATIN SMALL LETTER A, LAO VOWEL SIGN U, LAO TONE MAI EK, LAO VOWEL SIGN U, THAI CHARACTER MAI EK, LATIN SMALL LETTER B 0061 0EC8 0EB8 0E48 0EB9 0062;0061 0E48 0EB8 0EB9 0EC8 0062;0061 0E48 0EB8 0EB9 0EC8 0062;0061 0E48 0EB8 0EB9 0EC8 0062;0061 0E48 0EB8 0EB9 0EC8 0062; # (a◌່◌ຸ◌่◌ູb; a◌่◌ຸ◌ູ◌່b; a◌่◌ຸ◌ູ◌່b; a◌่◌ຸ◌ູ◌່b; a◌่◌ຸ◌ູ◌່b; ) LATIN SMALL LETTER A, LAO TONE MAI EK, LAO VOWEL SIGN U, THAI CHARACTER MAI EK, LAO VOWEL SIGN UU, LATIN SMALL LETTER B 0061 0EB9 0EC8 0EB8 0E48 0062;0061 0E48 0EB9 0EB8 0EC8 0062;0061 0E48 0EB9 0EB8 0EC8 0062;0061 0E48 0EB9 0EB8 0EC8 0062;0061 0E48 0EB9 0EB8 0EC8 0062; # (a◌ູ◌່◌ຸ◌่b; a◌่◌ູ◌ຸ◌່b; a◌่◌ູ◌ຸ◌່b; a◌่◌ູ◌ຸ◌່b; a◌่◌ູ◌ຸ◌່b; ) LATIN SMALL LETTER A, LAO VOWEL SIGN UU, LAO TONE MAI EK, LAO VOWEL SIGN U, THAI CHARACTER MAI EK, LATIN SMALL LETTER B +0061 05B0 094D 3099 0EBA 0062;0061 3099 094D 0EBA 05B0 0062;0061 3099 094D 0EBA 05B0 0062;0061 3099 094D 0EBA 05B0 0062;0061 3099 094D 0EBA 05B0 0062; # (a◌ְ◌्◌゙◌຺b; a◌゙◌्◌຺◌ְb; a◌゙◌्◌຺◌ְb; a◌゙◌्◌຺◌ְb; a◌゙◌्◌຺◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LAO SIGN PALI VIRAMA, LATIN SMALL LETTER B +0061 0EBA 05B0 094D 3099 0062;0061 3099 0EBA 094D 05B0 0062;0061 3099 0EBA 094D 05B0 0062;0061 3099 0EBA 094D 05B0 0062;0061 3099 0EBA 094D 05B0 0062; # (a◌຺◌ְ◌्◌゙b; a◌゙◌຺◌्◌ְb; a◌゙◌຺◌्◌ְb; a◌゙◌຺◌्◌ְb; a◌゙◌຺◌्◌ְb; ) LATIN SMALL LETTER A, LAO SIGN PALI VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B 0061 0F71 0EC8 0EB8 0EC8 0062;0061 0EB8 0EC8 0EC8 0F71 0062;0061 0EB8 0EC8 0EC8 0F71 0062;0061 0EB8 0EC8 0EC8 0F71 0062;0061 0EB8 0EC8 0EC8 0F71 0062; # (a◌ཱ◌່◌ຸ◌່b; a◌ຸ◌່◌່◌ཱb; a◌ຸ◌່◌່◌ཱb; a◌ຸ◌່◌່◌ཱb; a◌ຸ◌່◌່◌ཱb; ) LATIN SMALL LETTER A, TIBETAN VOWEL SIGN AA, LAO TONE MAI EK, LAO VOWEL SIGN U, LAO TONE MAI EK, LATIN SMALL LETTER B 0061 0EC8 0F71 0EC8 0EB8 0062;0061 0EB8 0EC8 0EC8 0F71 0062;0061 0EB8 0EC8 0EC8 0F71 0062;0061 0EB8 0EC8 0EC8 0F71 0062;0061 0EB8 0EC8 0EC8 0F71 0062; # (a◌່◌ཱ◌່◌ຸb; a◌ຸ◌່◌່◌ཱb; a◌ຸ◌່◌່◌ཱb; a◌ຸ◌່◌່◌ཱb; a◌ຸ◌່◌່◌ཱb; ) LATIN SMALL LETTER A, LAO TONE MAI EK, TIBETAN VOWEL SIGN AA, LAO TONE MAI EK, LAO VOWEL SIGN U, LATIN SMALL LETTER B 0061 0F71 0EC8 0EB8 0EC9 0062;0061 0EB8 0EC8 0EC9 0F71 0062;0061 0EB8 0EC8 0EC9 0F71 0062;0061 0EB8 0EC8 0EC9 0F71 0062;0061 0EB8 0EC8 0EC9 0F71 0062; # (a◌ཱ◌່◌ຸ◌້b; a◌ຸ◌່◌້◌ཱb; a◌ຸ◌່◌້◌ཱb; a◌ຸ◌່◌້◌ཱb; a◌ຸ◌່◌້◌ཱb; ) LATIN SMALL LETTER A, TIBETAN VOWEL SIGN AA, LAO TONE MAI EK, LAO VOWEL SIGN U, LAO TONE MAI THO, LATIN SMALL LETTER B @@ -18453,6 +18457,8 @@ FFEE;FFEE;FFEE;25CB;25CB; # (￮; ￮; ￮; ○; ○; ) HALFWIDTH WHITE CIRCLE 0061 11839 05B0 094D 3099 0062;0061 3099 11839 094D 05B0 0062;0061 3099 11839 094D 05B0 0062;0061 3099 11839 094D 05B0 0062;0061 3099 11839 094D 05B0 0062; # (a◌𑠹◌ְ◌्◌゙b; a◌゙◌𑠹◌्◌ְb; a◌゙◌𑠹◌्◌ְb; a◌゙◌𑠹◌्◌ְb; a◌゙◌𑠹◌्◌ְb; ) LATIN SMALL LETTER A, DOGRA SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B 0061 3099 093C 0334 1183A 0062;0061 0334 093C 1183A 3099 0062;0061 0334 093C 1183A 3099 0062;0061 0334 093C 1183A 3099 0062;0061 0334 093C 1183A 3099 0062; # (a◌゙◌़◌̴◌𑠺b; a◌̴◌़◌𑠺◌゙b; a◌̴◌़◌𑠺◌゙b; a◌̴◌़◌𑠺◌゙b; a◌̴◌़◌𑠺◌゙b; ) LATIN SMALL LETTER A, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, DOGRA SIGN NUKTA, LATIN SMALL LETTER B 0061 1183A 3099 093C 0334 0062;0061 0334 1183A 093C 3099 0062;0061 0334 1183A 093C 3099 0062;0061 0334 1183A 093C 3099 0062;0061 0334 1183A 093C 3099 0062; # (a◌𑠺◌゙◌़◌̴b; a◌̴◌𑠺◌़◌゙b; a◌̴◌𑠺◌़◌゙b; a◌̴◌𑠺◌़◌゙b; a◌̴◌𑠺◌़◌゙b; ) LATIN SMALL LETTER A, DOGRA SIGN NUKTA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B +0061 05B0 094D 3099 119E0 0062;0061 3099 094D 119E0 05B0 0062;0061 3099 094D 119E0 05B0 0062;0061 3099 094D 119E0 05B0 0062;0061 3099 094D 119E0 05B0 0062; # (a◌ְ◌्◌゙◌𑧠b; a◌゙◌्◌𑧠◌ְb; a◌゙◌्◌𑧠◌ְb; a◌゙◌्◌𑧠◌ְb; a◌゙◌्◌𑧠◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, NANDINAGARI SIGN VIRAMA, LATIN SMALL LETTER B +0061 119E0 05B0 094D 3099 0062;0061 3099 119E0 094D 05B0 0062;0061 3099 119E0 094D 05B0 0062;0061 3099 119E0 094D 05B0 0062;0061 3099 119E0 094D 05B0 0062; # (a◌𑧠◌ְ◌्◌゙b; a◌゙◌𑧠◌्◌ְb; a◌゙◌𑧠◌्◌ְb; a◌゙◌𑧠◌्◌ְb; a◌゙◌𑧠◌्◌ְb; ) LATIN SMALL LETTER A, NANDINAGARI SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B 0061 05B0 094D 3099 11A34 0062;0061 3099 094D 11A34 05B0 0062;0061 3099 094D 11A34 05B0 0062;0061 3099 094D 11A34 05B0 0062;0061 3099 094D 11A34 05B0 0062; # (a◌ְ◌्◌゙◌𑨴b; a◌゙◌्◌𑨴◌ְb; a◌゙◌्◌𑨴◌ְb; a◌゙◌्◌𑨴◌ְb; a◌゙◌्◌𑨴◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, ZANABAZAR SQUARE SIGN VIRAMA, LATIN SMALL LETTER B 0061 11A34 05B0 094D 3099 0062;0061 3099 11A34 094D 05B0 0062;0061 3099 11A34 094D 05B0 0062;0061 3099 11A34 094D 05B0 0062;0061 3099 11A34 094D 05B0 0062; # (a◌𑨴◌ְ◌्◌゙b; a◌゙◌𑨴◌्◌ְb; a◌゙◌𑨴◌्◌ְb; a◌゙◌𑨴◌्◌ְb; a◌゙◌𑨴◌्◌ְb; ) LATIN SMALL LETTER A, ZANABAZAR SQUARE SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B 0061 05B0 094D 3099 11A47 0062;0061 3099 094D 11A47 05B0 0062;0061 3099 094D 11A47 05B0 0062;0061 3099 094D 11A47 05B0 0062;0061 3099 094D 11A47 05B0 0062; # (a◌ְ◌्◌゙◌𑩇b; a◌゙◌्◌𑩇◌ְb; a◌゙◌्◌𑩇◌ְb; a◌゙◌्◌𑩇◌ְb; a◌゙◌्◌𑩇◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, ZANABAZAR SQUARE SUBJOINER, LATIN SMALL LETTER B @@ -18637,6 +18643,28 @@ FFEE;FFEE;FFEE;25CB;25CB; # (￮; ￮; ￮; ○; ○; ) HALFWIDTH WHITE CIRCLE 0061 1E029 0315 0300 05AE 0062;0061 05AE 1E029 0300 0315 0062;0061 05AE 1E029 0300 0315 0062;0061 05AE 1E029 0300 0315 0062;0061 05AE 1E029 0300 0315 0062; # (a◌𞀩◌̕◌̀◌֮b; a◌֮◌𞀩◌̀◌̕b; a◌֮◌𞀩◌̀◌̕b; a◌֮◌𞀩◌̀◌̕b; a◌֮◌𞀩◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING GLAGOLITIC LETTER IOTATED BIG YUS, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B 0061 0315 0300 05AE 1E02A 0062;00E0 05AE 1E02A 0315 0062;0061 05AE 0300 1E02A 0315 0062;00E0 05AE 1E02A 0315 0062;0061 05AE 0300 1E02A 0315 0062; # (a◌̕◌̀◌֮◌𞀪b; à◌֮◌𞀪◌̕b; a◌֮◌̀◌𞀪◌̕b; à◌֮◌𞀪◌̕b; a◌֮◌̀◌𞀪◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING GLAGOLITIC LETTER FITA, LATIN SMALL LETTER B 0061 1E02A 0315 0300 05AE 0062;0061 05AE 1E02A 0300 0315 0062;0061 05AE 1E02A 0300 0315 0062;0061 05AE 1E02A 0300 0315 0062;0061 05AE 1E02A 0300 0315 0062; # (a◌𞀪◌̕◌̀◌֮b; a◌֮◌𞀪◌̀◌̕b; a◌֮◌𞀪◌̀◌̕b; a◌֮◌𞀪◌̀◌̕b; a◌֮◌𞀪◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING GLAGOLITIC LETTER FITA, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 1E130 0062;00E0 05AE 1E130 0315 0062;0061 05AE 0300 1E130 0315 0062;00E0 05AE 1E130 0315 0062;0061 05AE 0300 1E130 0315 0062; # (a◌̕◌̀◌֮◌𞄰b; à◌֮◌𞄰◌̕b; a◌֮◌̀◌𞄰◌̕b; à◌֮◌𞄰◌̕b; a◌֮◌̀◌𞄰◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, NYIAKENG PUACHUE HMONG TONE-B, LATIN SMALL LETTER B +0061 1E130 0315 0300 05AE 0062;0061 05AE 1E130 0300 0315 0062;0061 05AE 1E130 0300 0315 0062;0061 05AE 1E130 0300 0315 0062;0061 05AE 1E130 0300 0315 0062; # (a◌𞄰◌̕◌̀◌֮b; a◌֮◌𞄰◌̀◌̕b; a◌֮◌𞄰◌̀◌̕b; a◌֮◌𞄰◌̀◌̕b; a◌֮◌𞄰◌̀◌̕b; ) LATIN SMALL LETTER A, NYIAKENG PUACHUE HMONG TONE-B, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 1E131 0062;00E0 05AE 1E131 0315 0062;0061 05AE 0300 1E131 0315 0062;00E0 05AE 1E131 0315 0062;0061 05AE 0300 1E131 0315 0062; # (a◌̕◌̀◌֮◌𞄱b; à◌֮◌𞄱◌̕b; a◌֮◌̀◌𞄱◌̕b; à◌֮◌𞄱◌̕b; a◌֮◌̀◌𞄱◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, NYIAKENG PUACHUE HMONG TONE-M, LATIN SMALL LETTER B +0061 1E131 0315 0300 05AE 0062;0061 05AE 1E131 0300 0315 0062;0061 05AE 1E131 0300 0315 0062;0061 05AE 1E131 0300 0315 0062;0061 05AE 1E131 0300 0315 0062; # (a◌𞄱◌̕◌̀◌֮b; a◌֮◌𞄱◌̀◌̕b; a◌֮◌𞄱◌̀◌̕b; a◌֮◌𞄱◌̀◌̕b; a◌֮◌𞄱◌̀◌̕b; ) LATIN SMALL LETTER A, NYIAKENG PUACHUE HMONG TONE-M, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 1E132 0062;00E0 05AE 1E132 0315 0062;0061 05AE 0300 1E132 0315 0062;00E0 05AE 1E132 0315 0062;0061 05AE 0300 1E132 0315 0062; # (a◌̕◌̀◌֮◌𞄲b; à◌֮◌𞄲◌̕b; a◌֮◌̀◌𞄲◌̕b; à◌֮◌𞄲◌̕b; a◌֮◌̀◌𞄲◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, NYIAKENG PUACHUE HMONG TONE-J, LATIN SMALL LETTER B +0061 1E132 0315 0300 05AE 0062;0061 05AE 1E132 0300 0315 0062;0061 05AE 1E132 0300 0315 0062;0061 05AE 1E132 0300 0315 0062;0061 05AE 1E132 0300 0315 0062; # (a◌𞄲◌̕◌̀◌֮b; a◌֮◌𞄲◌̀◌̕b; a◌֮◌𞄲◌̀◌̕b; a◌֮◌𞄲◌̀◌̕b; a◌֮◌𞄲◌̀◌̕b; ) LATIN SMALL LETTER A, NYIAKENG PUACHUE HMONG TONE-J, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 1E133 0062;00E0 05AE 1E133 0315 0062;0061 05AE 0300 1E133 0315 0062;00E0 05AE 1E133 0315 0062;0061 05AE 0300 1E133 0315 0062; # (a◌̕◌̀◌֮◌𞄳b; à◌֮◌𞄳◌̕b; a◌֮◌̀◌𞄳◌̕b; à◌֮◌𞄳◌̕b; a◌֮◌̀◌𞄳◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, NYIAKENG PUACHUE HMONG TONE-V, LATIN SMALL LETTER B +0061 1E133 0315 0300 05AE 0062;0061 05AE 1E133 0300 0315 0062;0061 05AE 1E133 0300 0315 0062;0061 05AE 1E133 0300 0315 0062;0061 05AE 1E133 0300 0315 0062; # (a◌𞄳◌̕◌̀◌֮b; a◌֮◌𞄳◌̀◌̕b; a◌֮◌𞄳◌̀◌̕b; a◌֮◌𞄳◌̀◌̕b; a◌֮◌𞄳◌̀◌̕b; ) LATIN SMALL LETTER A, NYIAKENG PUACHUE HMONG TONE-V, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 1E134 0062;00E0 05AE 1E134 0315 0062;0061 05AE 0300 1E134 0315 0062;00E0 05AE 1E134 0315 0062;0061 05AE 0300 1E134 0315 0062; # (a◌̕◌̀◌֮◌𞄴b; à◌֮◌𞄴◌̕b; a◌֮◌̀◌𞄴◌̕b; à◌֮◌𞄴◌̕b; a◌֮◌̀◌𞄴◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, NYIAKENG PUACHUE HMONG TONE-S, LATIN SMALL LETTER B +0061 1E134 0315 0300 05AE 0062;0061 05AE 1E134 0300 0315 0062;0061 05AE 1E134 0300 0315 0062;0061 05AE 1E134 0300 0315 0062;0061 05AE 1E134 0300 0315 0062; # (a◌𞄴◌̕◌̀◌֮b; a◌֮◌𞄴◌̀◌̕b; a◌֮◌𞄴◌̀◌̕b; a◌֮◌𞄴◌̀◌̕b; a◌֮◌𞄴◌̀◌̕b; ) LATIN SMALL LETTER A, NYIAKENG PUACHUE HMONG TONE-S, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 1E135 0062;00E0 05AE 1E135 0315 0062;0061 05AE 0300 1E135 0315 0062;00E0 05AE 1E135 0315 0062;0061 05AE 0300 1E135 0315 0062; # (a◌̕◌̀◌֮◌𞄵b; à◌֮◌𞄵◌̕b; a◌֮◌̀◌𞄵◌̕b; à◌֮◌𞄵◌̕b; a◌֮◌̀◌𞄵◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, NYIAKENG PUACHUE HMONG TONE-G, LATIN SMALL LETTER B +0061 1E135 0315 0300 05AE 0062;0061 05AE 1E135 0300 0315 0062;0061 05AE 1E135 0300 0315 0062;0061 05AE 1E135 0300 0315 0062;0061 05AE 1E135 0300 0315 0062; # (a◌𞄵◌̕◌̀◌֮b; a◌֮◌𞄵◌̀◌̕b; a◌֮◌𞄵◌̀◌̕b; a◌֮◌𞄵◌̀◌̕b; a◌֮◌𞄵◌̀◌̕b; ) LATIN SMALL LETTER A, NYIAKENG PUACHUE HMONG TONE-G, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 1E136 0062;00E0 05AE 1E136 0315 0062;0061 05AE 0300 1E136 0315 0062;00E0 05AE 1E136 0315 0062;0061 05AE 0300 1E136 0315 0062; # (a◌̕◌̀◌֮◌𞄶b; à◌֮◌𞄶◌̕b; a◌֮◌̀◌𞄶◌̕b; à◌֮◌𞄶◌̕b; a◌֮◌̀◌𞄶◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, NYIAKENG PUACHUE HMONG TONE-D, LATIN SMALL LETTER B +0061 1E136 0315 0300 05AE 0062;0061 05AE 1E136 0300 0315 0062;0061 05AE 1E136 0300 0315 0062;0061 05AE 1E136 0300 0315 0062;0061 05AE 1E136 0300 0315 0062; # (a◌𞄶◌̕◌̀◌֮b; a◌֮◌𞄶◌̀◌̕b; a◌֮◌𞄶◌̀◌̕b; a◌֮◌𞄶◌̀◌̕b; a◌֮◌𞄶◌̀◌̕b; ) LATIN SMALL LETTER A, NYIAKENG PUACHUE HMONG TONE-D, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 1E2EC 0062;00E0 05AE 1E2EC 0315 0062;0061 05AE 0300 1E2EC 0315 0062;00E0 05AE 1E2EC 0315 0062;0061 05AE 0300 1E2EC 0315 0062; # (a◌̕◌̀◌֮◌𞋬b; à◌֮◌𞋬◌̕b; a◌֮◌̀◌𞋬◌̕b; à◌֮◌𞋬◌̕b; a◌֮◌̀◌𞋬◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, WANCHO TONE TUP, LATIN SMALL LETTER B +0061 1E2EC 0315 0300 05AE 0062;0061 05AE 1E2EC 0300 0315 0062;0061 05AE 1E2EC 0300 0315 0062;0061 05AE 1E2EC 0300 0315 0062;0061 05AE 1E2EC 0300 0315 0062; # (a◌𞋬◌̕◌̀◌֮b; a◌֮◌𞋬◌̀◌̕b; a◌֮◌𞋬◌̀◌̕b; a◌֮◌𞋬◌̀◌̕b; a◌֮◌𞋬◌̀◌̕b; ) LATIN SMALL LETTER A, WANCHO TONE TUP, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 1E2ED 0062;00E0 05AE 1E2ED 0315 0062;0061 05AE 0300 1E2ED 0315 0062;00E0 05AE 1E2ED 0315 0062;0061 05AE 0300 1E2ED 0315 0062; # (a◌̕◌̀◌֮◌𞋭b; à◌֮◌𞋭◌̕b; a◌֮◌̀◌𞋭◌̕b; à◌֮◌𞋭◌̕b; a◌֮◌̀◌𞋭◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, WANCHO TONE TUPNI, LATIN SMALL LETTER B +0061 1E2ED 0315 0300 05AE 0062;0061 05AE 1E2ED 0300 0315 0062;0061 05AE 1E2ED 0300 0315 0062;0061 05AE 1E2ED 0300 0315 0062;0061 05AE 1E2ED 0300 0315 0062; # (a◌𞋭◌̕◌̀◌֮b; a◌֮◌𞋭◌̀◌̕b; a◌֮◌𞋭◌̀◌̕b; a◌֮◌𞋭◌̀◌̕b; a◌֮◌𞋭◌̀◌̕b; ) LATIN SMALL LETTER A, WANCHO TONE TUPNI, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 1E2EE 0062;00E0 05AE 1E2EE 0315 0062;0061 05AE 0300 1E2EE 0315 0062;00E0 05AE 1E2EE 0315 0062;0061 05AE 0300 1E2EE 0315 0062; # (a◌̕◌̀◌֮◌𞋮b; à◌֮◌𞋮◌̕b; a◌֮◌̀◌𞋮◌̕b; à◌֮◌𞋮◌̕b; a◌֮◌̀◌𞋮◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, WANCHO TONE KOI, LATIN SMALL LETTER B +0061 1E2EE 0315 0300 05AE 0062;0061 05AE 1E2EE 0300 0315 0062;0061 05AE 1E2EE 0300 0315 0062;0061 05AE 1E2EE 0300 0315 0062;0061 05AE 1E2EE 0300 0315 0062; # (a◌𞋮◌̕◌̀◌֮b; a◌֮◌𞋮◌̀◌̕b; a◌֮◌𞋮◌̀◌̕b; a◌֮◌𞋮◌̀◌̕b; a◌֮◌𞋮◌̀◌̕b; ) LATIN SMALL LETTER A, WANCHO TONE KOI, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 1E2EF 0062;00E0 05AE 1E2EF 0315 0062;0061 05AE 0300 1E2EF 0315 0062;00E0 05AE 1E2EF 0315 0062;0061 05AE 0300 1E2EF 0315 0062; # (a◌̕◌̀◌֮◌𞋯b; à◌֮◌𞋯◌̕b; a◌֮◌̀◌𞋯◌̕b; à◌֮◌𞋯◌̕b; a◌֮◌̀◌𞋯◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, WANCHO TONE KOINI, LATIN SMALL LETTER B +0061 1E2EF 0315 0300 05AE 0062;0061 05AE 1E2EF 0300 0315 0062;0061 05AE 1E2EF 0300 0315 0062;0061 05AE 1E2EF 0300 0315 0062;0061 05AE 1E2EF 0300 0315 0062; # (a◌𞋯◌̕◌̀◌֮b; a◌֮◌𞋯◌̀◌̕b; a◌֮◌𞋯◌̀◌̕b; a◌֮◌𞋯◌̀◌̕b; a◌֮◌𞋯◌̀◌̕b; ) LATIN SMALL LETTER A, WANCHO TONE KOINI, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B 0061 059A 0316 302A 1E8D0 0062;0061 302A 0316 1E8D0 059A 0062;0061 302A 0316 1E8D0 059A 0062;0061 302A 0316 1E8D0 059A 0062;0061 302A 0316 1E8D0 059A 0062; # (a◌֚◌̖◌〪◌𞣐b; a◌〪◌̖◌𞣐◌֚b; a◌〪◌̖◌𞣐◌֚b; a◌〪◌̖◌𞣐◌֚b; a◌〪◌̖◌𞣐◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, MENDE KIKAKUI COMBINING NUMBER TEENS, LATIN SMALL LETTER B 0061 1E8D0 059A 0316 302A 0062;0061 302A 1E8D0 0316 059A 0062;0061 302A 1E8D0 0316 059A 0062;0061 302A 1E8D0 0316 059A 0062;0061 302A 1E8D0 0316 059A 0062; # (a◌𞣐◌֚◌̖◌〪b; a◌〪◌𞣐◌̖◌֚b; a◌〪◌𞣐◌̖◌֚b; a◌〪◌𞣐◌̖◌֚b; a◌〪◌𞣐◌̖◌֚b; ) LATIN SMALL LETTER A, MENDE KIKAKUI COMBINING NUMBER TEENS, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B 0061 059A 0316 302A 1E8D1 0062;0061 302A 0316 1E8D1 059A 0062;0061 302A 0316 1E8D1 059A 0062;0061 302A 0316 1E8D1 059A 0062;0061 302A 0316 1E8D1 059A 0062; # (a◌֚◌̖◌〪◌𞣑b; a◌〪◌̖◌𞣑◌֚b; a◌〪◌̖◌𞣑◌֚b; a◌〪◌̖◌𞣑◌֚b; a◌〪◌̖◌𞣑◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, MENDE KIKAKUI COMBINING NUMBER TENS, LATIN SMALL LETTER B diff --git a/lib/stdlib/uc_spec/CaseFolding.txt b/lib/stdlib/uc_spec/CaseFolding.txt index cce350f49c..7eeb915abf 100644 --- a/lib/stdlib/uc_spec/CaseFolding.txt +++ b/lib/stdlib/uc_spec/CaseFolding.txt @@ -1,6 +1,6 @@ -# CaseFolding-11.0.0.txt -# Date: 2018-01-31, 08:20:09 GMT -# © 2018 Unicode®, Inc. +# CaseFolding-12.1.0.txt +# Date: 2019-03-10, 10:53:00 GMT +# © 2019 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use, see http://www.unicode.org/terms_of_use.html # @@ -1227,6 +1227,13 @@ A7B3; C; AB53; # LATIN CAPITAL LETTER CHI A7B4; C; A7B5; # LATIN CAPITAL LETTER BETA A7B6; C; A7B7; # LATIN CAPITAL LETTER OMEGA A7B8; C; A7B9; # LATIN CAPITAL LETTER U WITH STROKE +A7BA; C; A7BB; # LATIN CAPITAL LETTER GLOTTAL A +A7BC; C; A7BD; # LATIN CAPITAL LETTER GLOTTAL I +A7BE; C; A7BF; # LATIN CAPITAL LETTER GLOTTAL U +A7C2; C; A7C3; # LATIN CAPITAL LETTER ANGLICANA W +A7C4; C; A794; # LATIN CAPITAL LETTER C WITH PALATAL HOOK +A7C5; C; 0282; # LATIN CAPITAL LETTER S WITH HOOK +A7C6; C; 1D8E; # LATIN CAPITAL LETTER Z WITH PALATAL HOOK AB70; C; 13A0; # CHEROKEE SMALL LETTER A AB71; C; 13A1; # CHEROKEE SMALL LETTER E AB72; C; 13A2; # CHEROKEE SMALL LETTER I diff --git a/lib/stdlib/uc_spec/CompositionExclusions.txt b/lib/stdlib/uc_spec/CompositionExclusions.txt index ea63595bd3..aa654974be 100644 --- a/lib/stdlib/uc_spec/CompositionExclusions.txt +++ b/lib/stdlib/uc_spec/CompositionExclusions.txt @@ -1,6 +1,6 @@ -# CompositionExclusions-11.0.0.txt -# Date: 2017-12-06, 00:00:00 GMT [KW, LI] -# © 2017 Unicode®, Inc. +# CompositionExclusions-12.1.0.txt +# Date: 2019-03-08, 23:59:00 GMT [KW, LI] +# © 2019 Unicode®, Inc. # For terms of use, see http://www.unicode.org/terms_of_use.html # # Unicode Character Database diff --git a/lib/stdlib/uc_spec/GraphemeBreakProperty.txt b/lib/stdlib/uc_spec/GraphemeBreakProperty.txt index 52052e6e33..b75b201f97 100644 --- a/lib/stdlib/uc_spec/GraphemeBreakProperty.txt +++ b/lib/stdlib/uc_spec/GraphemeBreakProperty.txt @@ -1,6 +1,6 @@ -# GraphemeBreakProperty-11.0.0.txt -# Date: 2018-03-16, 20:34:02 GMT -# © 2018 Unicode®, Inc. +# GraphemeBreakProperty-12.1.0.txt +# Date: 2019-03-10, 10:53:12 GMT +# © 2019 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use, see http://www.unicode.org/terms_of_use.html # @@ -27,10 +27,10 @@ 110CD ; Prepend # Cf KAITHI NUMBER SIGN ABOVE 111C2..111C3 ; Prepend # Lo [2] SHARADA SIGN JIHVAMULIYA..SHARADA SIGN UPADHMANIYA 11A3A ; Prepend # Lo ZANABAZAR SQUARE CLUSTER-INITIAL LETTER RA -11A86..11A89 ; Prepend # Lo [4] SOYOMBO CLUSTER-INITIAL LETTER RA..SOYOMBO CLUSTER-INITIAL LETTER SA +11A84..11A89 ; Prepend # Lo [6] SOYOMBO SIGN JIHVAMULIYA..SOYOMBO CLUSTER-INITIAL LETTER SA 11D46 ; Prepend # Lo MASARAM GONDI REPHA -# Total code points: 20 +# Total code points: 22 # ================================================ @@ -61,10 +61,10 @@ 2060..2064 ; Control # Cf [5] WORD JOINER..INVISIBLE PLUS 2065 ; Control # Cn <reserved-2065> 2066..206F ; Control # Cf [10] LEFT-TO-RIGHT ISOLATE..NOMINAL DIGIT SHAPES -D800..DFFF ; Control # Cs [2048] <surrogate-D800>..<surrogate-DFFF> FEFF ; Control # Cf ZERO WIDTH NO-BREAK SPACE FFF0..FFF8 ; Control # Cn [9] <reserved-FFF0>..<reserved-FFF8> FFF9..FFFB ; Control # Cf [3] INTERLINEAR ANNOTATION ANCHOR..INTERLINEAR ANNOTATION TERMINATOR +13430..13438 ; Control # Cf [9] EGYPTIAN HIEROGLYPH VERTICAL JOINER..EGYPTIAN HIEROGLYPH END SEGMENT 1BCA0..1BCA3 ; Control # Cf [4] SHORTHAND FORMAT LETTER OVERLAP..SHORTHAND FORMAT UP STEP 1D173..1D17A ; Control # Cf [8] MUSICAL SYMBOL BEGIN BEAM..MUSICAL SYMBOL END PHRASE E0000 ; Control # Cn <reserved-E0000> @@ -73,7 +73,7 @@ E0002..E001F ; Control # Cn [30] <reserved-E0002>..<reserved-E001F> E0080..E00FF ; Control # Cn [128] <reserved-E0080>..<reserved-E00FF> E01F0..E0FFF ; Control # Cn [3600] <reserved-E01F0>..<reserved-E0FFF> -# Total code points: 5925 +# Total code points: 3886 # ================================================ @@ -178,8 +178,7 @@ E01F0..E0FFF ; Control # Cn [3600] <reserved-E01F0>..<reserved-E0FFF> 0E34..0E3A ; Extend # Mn [7] THAI CHARACTER SARA I..THAI CHARACTER PHINTHU 0E47..0E4E ; Extend # Mn [8] THAI CHARACTER MAITAIKHU..THAI CHARACTER YAMAKKAN 0EB1 ; Extend # Mn LAO VOWEL SIGN MAI KAN -0EB4..0EB9 ; Extend # Mn [6] LAO VOWEL SIGN I..LAO VOWEL SIGN UU -0EBB..0EBC ; Extend # Mn [2] LAO VOWEL SIGN MAI KON..LAO SEMIVOWEL SIGN LO +0EB4..0EBC ; Extend # Mn [9] LAO VOWEL SIGN I..LAO SEMIVOWEL SIGN LO 0EC8..0ECD ; Extend # Mn [6] LAO TONE MAI EK..LAO NIGGAHITA 0F18..0F19 ; Extend # Mn [2] TIBETAN ASTROLOGICAL SIGN -KHYUD PA..TIBETAN ASTROLOGICAL SIGN SDONG TSHUGS 0F35 ; Extend # Mn TIBETAN MARK NGAS BZUNG NYI ZLA @@ -232,6 +231,7 @@ E01F0..E0FFF ; Control # Cn [3600] <reserved-E01F0>..<reserved-E0FFF> 1ABE ; Extend # Me COMBINING PARENTHESES OVERLAY 1B00..1B03 ; Extend # Mn [4] BALINESE SIGN ULU RICEM..BALINESE SIGN SURANG 1B34 ; Extend # Mn BALINESE SIGN REREKAN +1B35 ; Extend # Mc BALINESE VOWEL SIGN TEDUNG 1B36..1B3A ; Extend # Mn [5] BALINESE VOWEL SIGN ULU..BALINESE VOWEL SIGN RA REPA 1B3C ; Extend # Mn BALINESE VOWEL SIGN LA LENGA 1B42 ; Extend # Mn BALINESE VOWEL SIGN PEPET @@ -283,7 +283,7 @@ A947..A951 ; Extend # Mn [11] REJANG VOWEL SIGN I..REJANG CONSONANT SIGN R A980..A982 ; Extend # Mn [3] JAVANESE SIGN PANYANGGA..JAVANESE SIGN LAYAR A9B3 ; Extend # Mn JAVANESE SIGN CECAK TELU A9B6..A9B9 ; Extend # Mn [4] JAVANESE VOWEL SIGN WULU..JAVANESE VOWEL SIGN SUKU MENDUT -A9BC ; Extend # Mn JAVANESE VOWEL SIGN PEPET +A9BC..A9BD ; Extend # Mn [2] JAVANESE VOWEL SIGN PEPET..JAVANESE CONSONANT SIGN KERET A9E5 ; Extend # Mn MYANMAR SIGN SHAN SAW AA29..AA2E ; Extend # Mn [6] CHAM VOWEL SIGN AA..CHAM VOWEL SIGN OE AA31..AA32 ; Extend # Mn [2] CHAM VOWEL SIGN AU..CHAM VOWEL SIGN UE @@ -368,6 +368,9 @@ FF9E..FF9F ; Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDT 11727..1172B ; Extend # Mn [5] AHOM VOWEL SIGN AW..AHOM SIGN KILLER 1182F..11837 ; Extend # Mn [9] DOGRA VOWEL SIGN U..DOGRA SIGN ANUSVARA 11839..1183A ; Extend # Mn [2] DOGRA SIGN VIRAMA..DOGRA SIGN NUKTA +119D4..119D7 ; Extend # Mn [4] NANDINAGARI VOWEL SIGN U..NANDINAGARI VOWEL SIGN VOCALIC RR +119DA..119DB ; Extend # Mn [2] NANDINAGARI VOWEL SIGN E..NANDINAGARI VOWEL SIGN AI +119E0 ; Extend # Mn NANDINAGARI SIGN VIRAMA 11A01..11A0A ; Extend # Mn [10] ZANABAZAR SQUARE VOWEL SIGN I..ZANABAZAR SQUARE VOWEL LENGTH MARK 11A33..11A38 ; Extend # Mn [6] ZANABAZAR SQUARE FINAL CONSONANT MARK..ZANABAZAR SQUARE SIGN ANUSVARA 11A3B..11A3E ; Extend # Mn [4] ZANABAZAR SQUARE CLUSTER-FINAL LETTER YA..ZANABAZAR SQUARE CLUSTER-FINAL LETTER VA @@ -394,6 +397,7 @@ FF9E..FF9F ; Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDT 11EF3..11EF4 ; Extend # Mn [2] MAKASAR VOWEL SIGN I..MAKASAR VOWEL SIGN U 16AF0..16AF4 ; Extend # Mn [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE 16B30..16B36 ; Extend # Mn [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM +16F4F ; Extend # Mn MIAO SIGN CONSONANT MODIFIER BAR 16F8F..16F92 ; Extend # Mn [4] MIAO TONE RIGHT..MIAO TONE BELOW 1BC9D..1BC9E ; Extend # Mn [2] DUPLOYAN THICK LETTER SELECTOR..DUPLOYAN DOUBLE MARK 1D165 ; Extend # Mc MUSICAL SYMBOL COMBINING STEM @@ -414,13 +418,15 @@ FF9E..FF9F ; Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDT 1E01B..1E021 ; Extend # Mn [7] COMBINING GLAGOLITIC LETTER SHTA..COMBINING GLAGOLITIC LETTER YATI 1E023..1E024 ; Extend # Mn [2] COMBINING GLAGOLITIC LETTER YU..COMBINING GLAGOLITIC LETTER SMALL YUS 1E026..1E02A ; Extend # Mn [5] COMBINING GLAGOLITIC LETTER YO..COMBINING GLAGOLITIC LETTER FITA +1E130..1E136 ; Extend # Mn [7] NYIAKENG PUACHUE HMONG TONE-B..NYIAKENG PUACHUE HMONG TONE-D +1E2EC..1E2EF ; Extend # Mn [4] WANCHO TONE TUP..WANCHO TONE KOINI 1E8D0..1E8D6 ; Extend # Mn [7] MENDE KIKAKUI COMBINING NUMBER TEENS..MENDE KIKAKUI COMBINING NUMBER MILLIONS 1E944..1E94A ; Extend # Mn [7] ADLAM ALIF LENGTHENER..ADLAM NUKTA 1F3FB..1F3FF ; Extend # Sk [5] EMOJI MODIFIER FITZPATRICK TYPE-1-2..EMOJI MODIFIER FITZPATRICK TYPE-6 E0020..E007F ; Extend # Cf [96] TAG SPACE..CANCEL TAG E0100..E01EF ; Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 -# Total code points: 1948 +# Total code points: 1970 # ================================================ @@ -489,7 +495,6 @@ E0100..E01EF ; Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 1A57 ; SpacingMark # Mc TAI THAM CONSONANT SIGN LA TANG LAI 1A6D..1A72 ; SpacingMark # Mc [6] TAI THAM VOWEL SIGN OY..TAI THAM VOWEL SIGN THAM AI 1B04 ; SpacingMark # Mc BALINESE SIGN BISAH -1B35 ; SpacingMark # Mc BALINESE VOWEL SIGN TEDUNG 1B3B ; SpacingMark # Mc BALINESE VOWEL SIGN RA REPA TEDUNG 1B3D..1B41 ; SpacingMark # Mc [5] BALINESE VOWEL SIGN LA LENGA TEDUNG..BALINESE VOWEL SIGN TALING REPA TEDUNG 1B43..1B44 ; SpacingMark # Mc [2] BALINESE VOWEL SIGN PEPET TEDUNG..BALINESE ADEG ADEG @@ -504,7 +509,6 @@ E0100..E01EF ; Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 1C24..1C2B ; SpacingMark # Mc [8] LEPCHA SUBJOINED LETTER YA..LEPCHA VOWEL SIGN UU 1C34..1C35 ; SpacingMark # Mc [2] LEPCHA CONSONANT SIGN NYIN-DO..LEPCHA CONSONANT SIGN KANG 1CE1 ; SpacingMark # Mc VEDIC TONE ATHARVAVEDIC INDEPENDENT SVARITA -1CF2..1CF3 ; SpacingMark # Mc [2] VEDIC SIGN ARDHAVISARGA..VEDIC SIGN ROTATED ARDHAVISARGA 1CF7 ; SpacingMark # Mc VEDIC SIGN ATIKRAMA A823..A824 ; SpacingMark # Mc [2] SYLOTI NAGRI VOWEL SIGN A..SYLOTI NAGRI VOWEL SIGN I A827 ; SpacingMark # Mc SYLOTI NAGRI VOWEL SIGN OO @@ -514,7 +518,7 @@ A952..A953 ; SpacingMark # Mc [2] REJANG CONSONANT SIGN H..REJANG VIRAMA A983 ; SpacingMark # Mc JAVANESE SIGN WIGNYAN A9B4..A9B5 ; SpacingMark # Mc [2] JAVANESE VOWEL SIGN TARUNG..JAVANESE VOWEL SIGN TOLONG A9BA..A9BB ; SpacingMark # Mc [2] JAVANESE VOWEL SIGN TALING..JAVANESE VOWEL SIGN DIRGA MURE -A9BD..A9C0 ; SpacingMark # Mc [4] JAVANESE CONSONANT SIGN KERET..JAVANESE PANGKON +A9BE..A9C0 ; SpacingMark # Mc [3] JAVANESE CONSONANT SIGN PENGKAL..JAVANESE PANGKON AA2F..AA30 ; SpacingMark # Mc [2] CHAM VOWEL SIGN O..CHAM VOWEL SIGN AI AA33..AA34 ; SpacingMark # Mc [2] CHAM CONSONANT SIGN YA..CHAM CONSONANT SIGN RA AA4D ; SpacingMark # Mc CHAM CONSONANT SIGN FINAL H @@ -566,6 +570,9 @@ ABEC ; SpacingMark # Mc MEETEI MAYEK LUM IYEK 11726 ; SpacingMark # Mc AHOM VOWEL SIGN E 1182C..1182E ; SpacingMark # Mc [3] DOGRA VOWEL SIGN AA..DOGRA VOWEL SIGN II 11838 ; SpacingMark # Mc DOGRA SIGN VISARGA +119D1..119D3 ; SpacingMark # Mc [3] NANDINAGARI VOWEL SIGN AA..NANDINAGARI VOWEL SIGN II +119DC..119DF ; SpacingMark # Mc [4] NANDINAGARI VOWEL SIGN O..NANDINAGARI SIGN VISARGA +119E4 ; SpacingMark # Mc NANDINAGARI VOWEL SIGN PRISHTHAMATRA E 11A39 ; SpacingMark # Mc ZANABAZAR SQUARE SIGN VISARGA 11A57..11A58 ; SpacingMark # Mc [2] SOYOMBO VOWEL SIGN AI..SOYOMBO VOWEL SIGN AU 11A97 ; SpacingMark # Mc SOYOMBO SIGN VISARGA @@ -578,11 +585,11 @@ ABEC ; SpacingMark # Mc MEETEI MAYEK LUM IYEK 11D93..11D94 ; SpacingMark # Mc [2] GUNJALA GONDI VOWEL SIGN OO..GUNJALA GONDI VOWEL SIGN AU 11D96 ; SpacingMark # Mc GUNJALA GONDI SIGN VISARGA 11EF5..11EF6 ; SpacingMark # Mc [2] MAKASAR VOWEL SIGN E..MAKASAR VOWEL SIGN O -16F51..16F7E ; SpacingMark # Mc [46] MIAO SIGN ASPIRATION..MIAO VOWEL SIGN NG +16F51..16F87 ; SpacingMark # Mc [55] MIAO SIGN ASPIRATION..MIAO VOWEL SIGN UI 1D166 ; SpacingMark # Mc MUSICAL SYMBOL COMBINING SPRECHGESANG STEM 1D16D ; SpacingMark # Mc MUSICAL SYMBOL COMBINING AUGMENTATION DOT -# Total code points: 362 +# Total code points: 375 # ================================================ diff --git a/lib/stdlib/uc_spec/PropList.txt b/lib/stdlib/uc_spec/PropList.txt index ef86795abe..4394602fea 100644 --- a/lib/stdlib/uc_spec/PropList.txt +++ b/lib/stdlib/uc_spec/PropList.txt @@ -1,6 +1,6 @@ -# PropList-11.0.0.txt -# Date: 2018-03-15, 04:28:35 GMT -# © 2018 Unicode®, Inc. +# PropList-12.1.0.txt +# Date: 2019-03-10, 10:53:16 GMT +# © 2019 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use, see http://www.unicode.org/terms_of_use.html # @@ -138,7 +138,7 @@ FF63 ; Quotation_Mark # Pe HALFWIDTH RIGHT CORNER BRACKET 0F0D..0F12 ; Terminal_Punctuation # Po [6] TIBETAN MARK SHAD..TIBETAN MARK RGYA GRAM SHAD 104A..104B ; Terminal_Punctuation # Po [2] MYANMAR SIGN LITTLE SECTION..MYANMAR SIGN SECTION 1361..1368 ; Terminal_Punctuation # Po [8] ETHIOPIC WORDSPACE..ETHIOPIC PARAGRAPH SEPARATOR -166D..166E ; Terminal_Punctuation # Po [2] CANADIAN SYLLABICS CHI SIGN..CANADIAN SYLLABICS FULL STOP +166E ; Terminal_Punctuation # Po CANADIAN SYLLABICS FULL STOP 16EB..16ED ; Terminal_Punctuation # Po [3] RUNIC SINGLE PUNCTUATION..RUNIC CROSS PUNCTUATION 1735..1736 ; Terminal_Punctuation # Po [2] PHILIPPINE SINGLE PUNCTUATION..PHILIPPINE DOUBLE PUNCTUATION 17D4..17D6 ; Terminal_Punctuation # Po [3] KHMER SIGN KHAN..KHMER SIGN CAMNUC PII KUUH @@ -157,7 +157,7 @@ FF63 ; Quotation_Mark # Pe HALFWIDTH RIGHT CORNER BRACKET 2E3C ; Terminal_Punctuation # Po STENOGRAPHIC FULL STOP 2E41 ; Terminal_Punctuation # Po REVERSED COMMA 2E4C ; Terminal_Punctuation # Po MEDIEVAL COMMA -2E4E ; Terminal_Punctuation # Po PUNCTUS ELEVATUS MARK +2E4E..2E4F ; Terminal_Punctuation # Po [2] PUNCTUS ELEVATUS MARK..CORNISH VERSE DIVIDER 3001..3002 ; Terminal_Punctuation # Po [2] IDEOGRAPHIC COMMA..IDEOGRAPHIC FULL STOP A4FE..A4FF ; Terminal_Punctuation # Po [2] LISU PUNCTUATION COMMA..LISU PUNCTUATION FULL STOP A60D..A60F ; Terminal_Punctuation # Po [3] VAI COMMA..VAI QUESTION MARK @@ -553,15 +553,17 @@ FF41..FF46 ; Hex_Digit # L& [6] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH L 1056..1057 ; Other_Alphabetic # Mc [2] MYANMAR VOWEL SIGN VOCALIC R..MYANMAR VOWEL SIGN VOCALIC RR 1058..1059 ; Other_Alphabetic # Mn [2] MYANMAR VOWEL SIGN VOCALIC L..MYANMAR VOWEL SIGN VOCALIC LL 105E..1060 ; Other_Alphabetic # Mn [3] MYANMAR CONSONANT SIGN MON MEDIAL NA..MYANMAR CONSONANT SIGN MON MEDIAL LA -1062 ; Other_Alphabetic # Mc MYANMAR VOWEL SIGN SGAW KAREN EU -1067..1068 ; Other_Alphabetic # Mc [2] MYANMAR VOWEL SIGN WESTERN PWO KAREN EU..MYANMAR VOWEL SIGN WESTERN PWO KAREN UE +1062..1064 ; Other_Alphabetic # Mc [3] MYANMAR VOWEL SIGN SGAW KAREN EU..MYANMAR TONE MARK SGAW KAREN KE PHO +1067..106D ; Other_Alphabetic # Mc [7] MYANMAR VOWEL SIGN WESTERN PWO KAREN EU..MYANMAR SIGN WESTERN PWO KAREN TONE-5 1071..1074 ; Other_Alphabetic # Mn [4] MYANMAR VOWEL SIGN GEBA KAREN I..MYANMAR VOWEL SIGN KAYAH EE 1082 ; Other_Alphabetic # Mn MYANMAR CONSONANT SIGN SHAN MEDIAL WA 1083..1084 ; Other_Alphabetic # Mc [2] MYANMAR VOWEL SIGN SHAN AA..MYANMAR VOWEL SIGN SHAN E 1085..1086 ; Other_Alphabetic # Mn [2] MYANMAR VOWEL SIGN SHAN E ABOVE..MYANMAR VOWEL SIGN SHAN FINAL Y -109C ; Other_Alphabetic # Mc MYANMAR VOWEL SIGN AITON A +1087..108C ; Other_Alphabetic # Mc [6] MYANMAR SIGN SHAN TONE-2..MYANMAR SIGN SHAN COUNCIL TONE-3 +108D ; Other_Alphabetic # Mn MYANMAR SIGN SHAN COUNCIL EMPHATIC TONE +108F ; Other_Alphabetic # Mc MYANMAR SIGN RUMAI PALAUNG TONE-5 +109A..109C ; Other_Alphabetic # Mc [3] MYANMAR SIGN KHAMTI TONE-1..MYANMAR VOWEL SIGN AITON A 109D ; Other_Alphabetic # Mn MYANMAR VOWEL SIGN AITON AI -135F ; Other_Alphabetic # Mn ETHIOPIC COMBINING GEMINATION MARK 1712..1713 ; Other_Alphabetic # Mn [2] TAGALOG VOWEL SIGN I..TAGALOG VOWEL SIGN U 1732..1733 ; Other_Alphabetic # Mn [2] HANUNOO VOWEL SIGN I..HANUNOO VOWEL SIGN U 1752..1753 ; Other_Alphabetic # Mn [2] BUHID VOWEL SIGN I..BUHID VOWEL SIGN U @@ -618,18 +620,21 @@ FF41..FF46 ; Hex_Digit # L& [6] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH L 1C24..1C2B ; Other_Alphabetic # Mc [8] LEPCHA SUBJOINED LETTER YA..LEPCHA VOWEL SIGN UU 1C2C..1C33 ; Other_Alphabetic # Mn [8] LEPCHA VOWEL SIGN E..LEPCHA CONSONANT SIGN T 1C34..1C35 ; Other_Alphabetic # Mc [2] LEPCHA CONSONANT SIGN NYIN-DO..LEPCHA CONSONANT SIGN KANG -1CF2..1CF3 ; Other_Alphabetic # Mc [2] VEDIC SIGN ARDHAVISARGA..VEDIC SIGN ROTATED ARDHAVISARGA +1C36 ; Other_Alphabetic # Mn LEPCHA SIGN RAN 1DE7..1DF4 ; Other_Alphabetic # Mn [14] COMBINING LATIN SMALL LETTER ALPHA..COMBINING LATIN SMALL LETTER U WITH DIAERESIS 24B6..24E9 ; Other_Alphabetic # So [52] CIRCLED LATIN CAPITAL LETTER A..CIRCLED LATIN SMALL LETTER Z 2DE0..2DFF ; Other_Alphabetic # Mn [32] COMBINING CYRILLIC LETTER BE..COMBINING CYRILLIC LETTER IOTIFIED BIG YUS A674..A67B ; Other_Alphabetic # Mn [8] COMBINING CYRILLIC LETTER UKRAINIAN IE..COMBINING CYRILLIC LETTER OMEGA A69E..A69F ; Other_Alphabetic # Mn [2] COMBINING CYRILLIC LETTER EF..COMBINING CYRILLIC LETTER IOTIFIED E +A802 ; Other_Alphabetic # Mn SYLOTI NAGRI SIGN DVISVARA +A80B ; Other_Alphabetic # Mn SYLOTI NAGRI SIGN ANUSVARA A823..A824 ; Other_Alphabetic # Mc [2] SYLOTI NAGRI VOWEL SIGN A..SYLOTI NAGRI VOWEL SIGN I A825..A826 ; Other_Alphabetic # Mn [2] SYLOTI NAGRI VOWEL SIGN U..SYLOTI NAGRI VOWEL SIGN E A827 ; Other_Alphabetic # Mc SYLOTI NAGRI VOWEL SIGN OO A880..A881 ; Other_Alphabetic # Mc [2] SAURASHTRA SIGN ANUSVARA..SAURASHTRA SIGN VISARGA A8B4..A8C3 ; Other_Alphabetic # Mc [16] SAURASHTRA CONSONANT SIGN HAARU..SAURASHTRA VOWEL SIGN AU A8C5 ; Other_Alphabetic # Mn SAURASHTRA SIGN CANDRABINDU +A8FF ; Other_Alphabetic # Mn DEVANAGARI VOWEL SIGN AY A926..A92A ; Other_Alphabetic # Mn [5] KAYAH LI VOWEL UE..KAYAH LI VOWEL O A947..A951 ; Other_Alphabetic # Mn [11] REJANG VOWEL SIGN I..REJANG CONSONANT SIGN R A952 ; Other_Alphabetic # Mc REJANG CONSONANT SIGN H @@ -638,8 +643,9 @@ A983 ; Other_Alphabetic # Mc JAVANESE SIGN WIGNYAN A9B4..A9B5 ; Other_Alphabetic # Mc [2] JAVANESE VOWEL SIGN TARUNG..JAVANESE VOWEL SIGN TOLONG A9B6..A9B9 ; Other_Alphabetic # Mn [4] JAVANESE VOWEL SIGN WULU..JAVANESE VOWEL SIGN SUKU MENDUT A9BA..A9BB ; Other_Alphabetic # Mc [2] JAVANESE VOWEL SIGN TALING..JAVANESE VOWEL SIGN DIRGA MURE -A9BC ; Other_Alphabetic # Mn JAVANESE VOWEL SIGN PEPET -A9BD..A9BF ; Other_Alphabetic # Mc [3] JAVANESE CONSONANT SIGN KERET..JAVANESE CONSONANT SIGN CAKRA +A9BC..A9BD ; Other_Alphabetic # Mn [2] JAVANESE VOWEL SIGN PEPET..JAVANESE CONSONANT SIGN KERET +A9BE..A9BF ; Other_Alphabetic # Mc [2] JAVANESE CONSONANT SIGN PENGKAL..JAVANESE CONSONANT SIGN CAKRA +A9E5 ; Other_Alphabetic # Mn MYANMAR SIGN SHAN SAW AA29..AA2E ; Other_Alphabetic # Mn [6] CHAM VOWEL SIGN AA..CHAM VOWEL SIGN OE AA2F..AA30 ; Other_Alphabetic # Mc [2] CHAM VOWEL SIGN O..CHAM VOWEL SIGN AI AA31..AA32 ; Other_Alphabetic # Mn [2] CHAM VOWEL SIGN AU..CHAM VOWEL SIGN UE @@ -648,6 +654,9 @@ AA35..AA36 ; Other_Alphabetic # Mn [2] CHAM CONSONANT SIGN LA..CHAM CONSONA AA43 ; Other_Alphabetic # Mn CHAM CONSONANT SIGN FINAL NG AA4C ; Other_Alphabetic # Mn CHAM CONSONANT SIGN FINAL M AA4D ; Other_Alphabetic # Mc CHAM CONSONANT SIGN FINAL H +AA7B ; Other_Alphabetic # Mc MYANMAR SIGN PAO KAREN TONE +AA7C ; Other_Alphabetic # Mn MYANMAR SIGN TAI LAING TONE-2 +AA7D ; Other_Alphabetic # Mc MYANMAR SIGN TAI LAING TONE-5 AAB0 ; Other_Alphabetic # Mn TAI VIET MAI KANG AAB2..AAB4 ; Other_Alphabetic # Mn [3] TAI VIET VOWEL I..TAI VIET VOWEL U AAB7..AAB8 ; Other_Alphabetic # Mn [2] TAI VIET MAI KHIT..TAI VIET VOWEL IA @@ -740,6 +749,11 @@ FB1E ; Other_Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA 1182C..1182E ; Other_Alphabetic # Mc [3] DOGRA VOWEL SIGN AA..DOGRA VOWEL SIGN II 1182F..11837 ; Other_Alphabetic # Mn [9] DOGRA VOWEL SIGN U..DOGRA SIGN ANUSVARA 11838 ; Other_Alphabetic # Mc DOGRA SIGN VISARGA +119D1..119D3 ; Other_Alphabetic # Mc [3] NANDINAGARI VOWEL SIGN AA..NANDINAGARI VOWEL SIGN II +119D4..119D7 ; Other_Alphabetic # Mn [4] NANDINAGARI VOWEL SIGN U..NANDINAGARI VOWEL SIGN VOCALIC RR +119DA..119DB ; Other_Alphabetic # Mn [2] NANDINAGARI VOWEL SIGN E..NANDINAGARI VOWEL SIGN AI +119DC..119DF ; Other_Alphabetic # Mc [4] NANDINAGARI VOWEL SIGN O..NANDINAGARI SIGN VISARGA +119E4 ; Other_Alphabetic # Mc NANDINAGARI VOWEL SIGN PRISHTHAMATRA E 11A01..11A0A ; Other_Alphabetic # Mn [10] ZANABAZAR SQUARE VOWEL SIGN I..ZANABAZAR SQUARE VOWEL LENGTH MARK 11A35..11A38 ; Other_Alphabetic # Mn [4] ZANABAZAR SQUARE SIGN CANDRABINDU..ZANABAZAR SQUARE SIGN ANUSVARA 11A39 ; Other_Alphabetic # Mc ZANABAZAR SQUARE SIGN VISARGA @@ -773,8 +787,9 @@ FB1E ; Other_Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA 11D96 ; Other_Alphabetic # Mc GUNJALA GONDI SIGN VISARGA 11EF3..11EF4 ; Other_Alphabetic # Mn [2] MAKASAR VOWEL SIGN I..MAKASAR VOWEL SIGN U 11EF5..11EF6 ; Other_Alphabetic # Mc [2] MAKASAR VOWEL SIGN E..MAKASAR VOWEL SIGN O -16B30..16B36 ; Other_Alphabetic # Mn [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM -16F51..16F7E ; Other_Alphabetic # Mc [46] MIAO SIGN ASPIRATION..MIAO VOWEL SIGN NG +16F4F ; Other_Alphabetic # Mn MIAO SIGN CONSONANT MODIFIER BAR +16F51..16F87 ; Other_Alphabetic # Mc [55] MIAO SIGN ASPIRATION..MIAO VOWEL SIGN UI +16F8F..16F92 ; Other_Alphabetic # Mn [4] MIAO TONE RIGHT..MIAO TONE BELOW 1BC9E ; Other_Alphabetic # Mn DUPLOYAN DOUBLE MARK 1E000..1E006 ; Other_Alphabetic # Mn [7] COMBINING GLAGOLITIC LETTER AZU..COMBINING GLAGOLITIC LETTER ZHIVETE 1E008..1E018 ; Other_Alphabetic # Mn [17] COMBINING GLAGOLITIC LETTER ZEMLJA..COMBINING GLAGOLITIC LETTER HERU @@ -786,7 +801,7 @@ FB1E ; Other_Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA 1F150..1F169 ; Other_Alphabetic # So [26] NEGATIVE CIRCLED LATIN CAPITAL LETTER A..NEGATIVE CIRCLED LATIN CAPITAL LETTER Z 1F170..1F189 ; Other_Alphabetic # So [26] NEGATIVE SQUARED LATIN CAPITAL LETTER A..NEGATIVE SQUARED LATIN CAPITAL LETTER Z -# Total code points: 1334 +# Total code points: 1377 # ================================================ @@ -798,7 +813,7 @@ FB1E ; Other_Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA 4E00..9FEF ; Ideographic # Lo [20976] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FEF F900..FA6D ; Ideographic # Lo [366] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA6D FA70..FAD9 ; Ideographic # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILITY IDEOGRAPH-FAD9 -17000..187F1 ; Ideographic # Lo [6130] TANGUT IDEOGRAPH-17000..TANGUT IDEOGRAPH-187F1 +17000..187F7 ; Ideographic # Lo [6136] TANGUT IDEOGRAPH-17000..TANGUT IDEOGRAPH-187F7 18800..18AF2 ; Ideographic # Lo [755] TANGUT COMPONENT-001..TANGUT COMPONENT-755 1B170..1B2FB ; Ideographic # Lo [396] NUSHU CHARACTER-1B170..NUSHU CHARACTER-1B2FB 20000..2A6D6 ; Ideographic # Lo [42711] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6D6 @@ -808,7 +823,7 @@ FA70..FAD9 ; Ideographic # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COM 2CEB0..2EBE0 ; Ideographic # Lo [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0 2F800..2FA1D ; Ideographic # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D -# Total code points: 96184 +# Total code points: 96190 # ================================================ @@ -876,6 +891,7 @@ FA70..FAD9 ; Ideographic # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COM 0DCA ; Diacritic # Mn SINHALA SIGN AL-LAKUNA 0E47..0E4C ; Diacritic # Mn [6] THAI CHARACTER MAITAIKHU..THAI CHARACTER THANTHAKHAT 0E4E ; Diacritic # Mn THAI CHARACTER YAMAKKAN +0EBA ; Diacritic # Mn LAO SIGN PALI VIRAMA 0EC8..0ECC ; Diacritic # Mn [5] LAO TONE MAI EK..LAO CANCELLATION MARK 0F18..0F19 ; Diacritic # Mn [2] TIBETAN ASTROLOGICAL SIGN -KHYUD PA..TIBETAN ASTROLOGICAL SIGN SDONG TSHUGS 0F35 ; Diacritic # Mn TIBETAN MARK NGAS BZUNG NYI ZLA @@ -887,10 +903,13 @@ FA70..FAD9 ; Ideographic # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COM 0FC6 ; Diacritic # Mn TIBETAN SYMBOL PADMA GDAN 1037 ; Diacritic # Mn MYANMAR SIGN DOT BELOW 1039..103A ; Diacritic # Mn [2] MYANMAR SIGN VIRAMA..MYANMAR SIGN ASAT +1063..1064 ; Diacritic # Mc [2] MYANMAR TONE MARK SGAW KAREN HATHI..MYANMAR TONE MARK SGAW KAREN KE PHO +1069..106D ; Diacritic # Mc [5] MYANMAR SIGN WESTERN PWO KAREN TONE-1..MYANMAR SIGN WESTERN PWO KAREN TONE-5 1087..108C ; Diacritic # Mc [6] MYANMAR SIGN SHAN TONE-2..MYANMAR SIGN SHAN COUNCIL TONE-3 108D ; Diacritic # Mn MYANMAR SIGN SHAN COUNCIL EMPHATIC TONE 108F ; Diacritic # Mc MYANMAR SIGN RUMAI PALAUNG TONE-5 109A..109B ; Diacritic # Mc [2] MYANMAR SIGN KHAMTI TONE-1..MYANMAR SIGN KHAMTI TONE-3 +135D..135F ; Diacritic # Mn [3] ETHIOPIC COMBINING GEMINATION AND VOWEL LENGTH MARK..ETHIOPIC COMBINING GEMINATION MARK 17C9..17D3 ; Diacritic # Mn [11] KHMER SIGN MUUSIKATOAN..KHMER SIGN BATHAMASAT 17DD ; Diacritic # Mn KHMER SIGN ATTHACAN 1939..193B ; Diacritic # Mn [3] LIMBU SIGN MUKPHRENG..LIMBU SIGN SA-I @@ -935,9 +954,11 @@ A67C..A67D ; Diacritic # Mn [2] COMBINING CYRILLIC KAVYKA..COMBINING CYRILL A67F ; Diacritic # Lm CYRILLIC PAYEROK A69C..A69D ; Diacritic # Lm [2] MODIFIER LETTER CYRILLIC HARD SIGN..MODIFIER LETTER CYRILLIC SOFT SIGN A6F0..A6F1 ; Diacritic # Mn [2] BAMUM COMBINING MARK KOQNDON..BAMUM COMBINING MARK TUKWENTIS +A700..A716 ; Diacritic # Sk [23] MODIFIER LETTER CHINESE TONE YIN PING..MODIFIER LETTER EXTRA-LOW LEFT-STEM TONE BAR A717..A71F ; Diacritic # Lm [9] MODIFIER LETTER DOT VERTICAL BAR..MODIFIER LETTER LOW INVERTED EXCLAMATION MARK A720..A721 ; Diacritic # Sk [2] MODIFIER LETTER STRESS AND HIGH TONE..MODIFIER LETTER STRESS AND LOW TONE A788 ; Diacritic # Lm MODIFIER LETTER LOW CIRCUMFLEX ACCENT +A789..A78A ; Diacritic # Sk [2] MODIFIER LETTER COLON..MODIFIER LETTER SHORT EQUALS SIGN A7F8..A7F9 ; Diacritic # Lm [2] MODIFIER LETTER CAPITAL H WITH STROKE..MODIFIER LETTER SMALL LIGATURE OE A8C4 ; Diacritic # Mn SAURASHTRA SIGN VIRAMA A8E0..A8F1 ; Diacritic # Mn [18] COMBINING DEVANAGARI DIGIT ZERO..COMBINING DEVANAGARI SIGN AVAGRAHA @@ -992,6 +1013,7 @@ FFE3 ; Diacritic # Sk FULLWIDTH MACRON 116B7 ; Diacritic # Mn TAKRI SIGN NUKTA 1172B ; Diacritic # Mn AHOM SIGN KILLER 11839..1183A ; Diacritic # Mn [2] DOGRA SIGN VIRAMA..DOGRA SIGN NUKTA +119E0 ; Diacritic # Mn NANDINAGARI SIGN VIRAMA 11A34 ; Diacritic # Mn ZANABAZAR SQUARE SIGN VIRAMA 11A47 ; Diacritic # Mn ZANABAZAR SQUARE SUBJOINER 11A99 ; Diacritic # Mn SOYOMBO SUBJOINER @@ -1000,6 +1022,7 @@ FFE3 ; Diacritic # Sk FULLWIDTH MACRON 11D44..11D45 ; Diacritic # Mn [2] MASARAM GONDI SIGN HALANTA..MASARAM GONDI VIRAMA 11D97 ; Diacritic # Mn GUNJALA GONDI VIRAMA 16AF0..16AF4 ; Diacritic # Mn [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE +16B30..16B36 ; Diacritic # Mn [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM 16F8F..16F92 ; Diacritic # Mn [4] MIAO TONE RIGHT..MIAO TONE BELOW 16F93..16F9F ; Diacritic # Lm [13] MIAO LETTER TONE-2..MIAO LETTER REFORMED TONE-8 1D167..1D169 ; Diacritic # Mn [3] MUSICAL SYMBOL COMBINING TREMOLO-1..MUSICAL SYMBOL COMBINING TREMOLO-3 @@ -1007,11 +1030,13 @@ FFE3 ; Diacritic # Sk FULLWIDTH MACRON 1D17B..1D182 ; Diacritic # Mn [8] MUSICAL SYMBOL COMBINING ACCENT..MUSICAL SYMBOL COMBINING LOURE 1D185..1D18B ; Diacritic # Mn [7] MUSICAL SYMBOL COMBINING DOIT..MUSICAL SYMBOL COMBINING TRIPLE TONGUE 1D1AA..1D1AD ; Diacritic # Mn [4] MUSICAL SYMBOL COMBINING DOWN BOW..MUSICAL SYMBOL COMBINING SNAP PIZZICATO +1E130..1E136 ; Diacritic # Mn [7] NYIAKENG PUACHUE HMONG TONE-B..NYIAKENG PUACHUE HMONG TONE-D +1E2EC..1E2EF ; Diacritic # Mn [4] WANCHO TONE TUP..WANCHO TONE KOINI 1E8D0..1E8D6 ; Diacritic # Mn [7] MENDE KIKAKUI COMBINING NUMBER TEENS..MENDE KIKAKUI COMBINING NUMBER MILLIONS 1E944..1E946 ; Diacritic # Mn [3] ADLAM ALIF LENGTHENER..ADLAM GEMINATION MARK 1E948..1E94A ; Diacritic # Mn [3] ADLAM CONSONANT MODIFIER..ADLAM NUKTA -# Total code points: 818 +# Total code points: 873 # ================================================ @@ -1043,9 +1068,11 @@ FF70 ; Extender # Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND 11A98 ; Extender # Mn SOYOMBO GEMINATION MARK 16B42..16B43 ; Extender # Lm [2] PAHAWH HMONG SIGN VOS NRUA..PAHAWH HMONG SIGN IB YAM 16FE0..16FE1 ; Extender # Lm [2] TANGUT ITERATION MARK..NUSHU ITERATION MARK +16FE3 ; Extender # Lm OLD CHINESE ITERATION MARK +1E13C..1E13D ; Extender # Lm [2] NYIAKENG PUACHUE HMONG SIGN XW XW..NYIAKENG PUACHUE HMONG SYLLABLE LENGTHENER 1E944..1E946 ; Extender # Mn [3] ADLAM ALIF LENGTHENER..ADLAM GEMINATION MARK -# Total code points: 44 +# Total code points: 47 # ================================================ @@ -1119,6 +1146,7 @@ FFFFE..FFFFF ; Noncharacter_Code_Point # Cn [2] <noncharacter-FFFFE>..<noncha 0D57 ; Other_Grapheme_Extend # Mc MALAYALAM AU LENGTH MARK 0DCF ; Other_Grapheme_Extend # Mc SINHALA VOWEL SIGN AELA-PILLA 0DDF ; Other_Grapheme_Extend # Mc SINHALA VOWEL SIGN GAYANUKITTA +1B35 ; Other_Grapheme_Extend # Mc BALINESE VOWEL SIGN TEDUNG 200C ; Other_Grapheme_Extend # Cf ZERO WIDTH NON-JOINER 302E..302F ; Other_Grapheme_Extend # Mc [2] HANGUL SINGLE DOT TONE MARK..HANGUL DOUBLE DOT TONE MARK FF9E..FF9F ; Other_Grapheme_Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK @@ -1131,7 +1159,7 @@ FF9E..FF9F ; Other_Grapheme_Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND 1D16E..1D172 ; Other_Grapheme_Extend # Mc [5] MUSICAL SYMBOL COMBINING FLAG-1..MUSICAL SYMBOL COMBINING FLAG-5 E0020..E007F ; Other_Grapheme_Extend # Cf [96] TAG SPACE..CANCEL TAG -# Total code points: 125 +# Total code points: 126 # ================================================ @@ -1547,10 +1575,7 @@ E0100..E01EF ; Variation_Selector # Mn [240] VARIATION SELECTOR-17..VARIATION S 2B74..2B75 ; Pattern_Syntax # Cn [2] <reserved-2B74>..<reserved-2B75> 2B76..2B95 ; Pattern_Syntax # So [32] NORTH WEST TRIANGLE-HEADED ARROW TO BAR..RIGHTWARDS BLACK ARROW 2B96..2B97 ; Pattern_Syntax # Cn [2] <reserved-2B96>..<reserved-2B97> -2B98..2BC8 ; Pattern_Syntax # So [49] THREE-D TOP-LIGHTED LEFTWARDS EQUILATERAL ARROWHEAD..BLACK MEDIUM RIGHT-POINTING TRIANGLE CENTRED -2BC9 ; Pattern_Syntax # Cn <reserved-2BC9> -2BCA..2BFE ; Pattern_Syntax # So [53] TOP HALF BLACK CIRCLE..REVERSED RIGHT ANGLE -2BFF ; Pattern_Syntax # Cn <reserved-2BFF> +2B98..2BFF ; Pattern_Syntax # So [104] THREE-D TOP-LIGHTED LEFTWARDS EQUILATERAL ARROWHEAD..HELLSCHREIBER PAUSE SYMBOL 2E00..2E01 ; Pattern_Syntax # Po [2] RIGHT ANGLE SUBSTITUTION MARKER..RIGHT ANGLE DOTTED SUBSTITUTION MARKER 2E02 ; Pattern_Syntax # Pi LEFT SUBSTITUTION BRACKET 2E03 ; Pattern_Syntax # Pf RIGHT SUBSTITUTION BRACKET @@ -1588,8 +1613,8 @@ E0100..E01EF ; Variation_Selector # Mn [240] VARIATION SELECTOR-17..VARIATION S 2E40 ; Pattern_Syntax # Pd DOUBLE HYPHEN 2E41 ; Pattern_Syntax # Po REVERSED COMMA 2E42 ; Pattern_Syntax # Ps DOUBLE LOW-REVERSED-9 QUOTATION MARK -2E43..2E4E ; Pattern_Syntax # Po [12] DASH WITH LEFT UPTURN..PUNCTUS ELEVATUS MARK -2E4F..2E7F ; Pattern_Syntax # Cn [49] <reserved-2E4F>..<reserved-2E7F> +2E43..2E4F ; Pattern_Syntax # Po [13] DASH WITH LEFT UPTURN..CORNISH VERSE DIVIDER +2E50..2E7F ; Pattern_Syntax # Cn [48] <reserved-2E50>..<reserved-2E7F> 3001..3003 ; Pattern_Syntax # Po [3] IDEOGRAPHIC COMMA..DITTO MARK 3008 ; Pattern_Syntax # Ps LEFT ANGLE BRACKET 3009 ; Pattern_Syntax # Pe RIGHT ANGLE BRACKET diff --git a/lib/stdlib/uc_spec/README-UPDATE.txt b/lib/stdlib/uc_spec/README-UPDATE.txt index e1f5c8fcd0..73274e512a 100644 --- a/lib/stdlib/uc_spec/README-UPDATE.txt +++ b/lib/stdlib/uc_spec/README-UPDATE.txt @@ -2,12 +2,10 @@ When updating the unicode version copy the necessary files to this directory. And update the test files in stdlib/test/unicode_util_SUITE_data/* -Unicode 11 was updated from: -https://www.unicode.org/Public/11.0.0/ucd/ -https://www.unicode.org/Public/11.0.0/ucd/auxiliary/ -https://www.unicode.org/Public/emoji/11.0/ +Unicode 12.1 was updated from: +https://www.unicode.org/Public/12.1.0/ucd/ +https://www.unicode.org/Public/12.1.0/ucd/auxiliary/ +https://www.unicode.org/Public/emoji/12.0/ Update the spec_version(..) function in the generator, gen_unicode_mod.escript - - diff --git a/lib/stdlib/uc_spec/SpecialCasing.txt b/lib/stdlib/uc_spec/SpecialCasing.txt index c90d09acb3..1c04aacf97 100644 --- a/lib/stdlib/uc_spec/SpecialCasing.txt +++ b/lib/stdlib/uc_spec/SpecialCasing.txt @@ -1,6 +1,6 @@ -# SpecialCasing-11.0.0.txt -# Date: 2018-02-22, 06:16:47 GMT -# © 2018 Unicode®, Inc. +# SpecialCasing-12.1.0.txt +# Date: 2019-03-10, 10:53:28 GMT +# © 2019 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use, see http://www.unicode.org/terms_of_use.html # diff --git a/lib/stdlib/uc_spec/UnicodeData.txt b/lib/stdlib/uc_spec/UnicodeData.txt index ec32fafbce..e65aec52f7 100644 --- a/lib/stdlib/uc_spec/UnicodeData.txt +++ b/lib/stdlib/uc_spec/UnicodeData.txt @@ -640,7 +640,7 @@ 027F;LATIN SMALL LETTER REVERSED R WITH FISHHOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER REVERSED FISHHOOK R;;;; 0280;LATIN LETTER SMALL CAPITAL R;Ll;0;L;;;;;N;;;01A6;;01A6 0281;LATIN LETTER SMALL CAPITAL INVERTED R;Ll;0;L;;;;;N;;;;; -0282;LATIN SMALL LETTER S WITH HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER S HOOK;;;; +0282;LATIN SMALL LETTER S WITH HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER S HOOK;;A7C5;;A7C5 0283;LATIN SMALL LETTER ESH;Ll;0;L;;;;;N;;;01A9;;01A9 0284;LATIN SMALL LETTER DOTLESS J WITH STROKE AND HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER DOTLESS J BAR HOOK;;;; 0285;LATIN SMALL LETTER SQUAT REVERSED ESH;Ll;0;L;;;;;N;;;;; @@ -2809,6 +2809,7 @@ 0C6D;TELUGU DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; 0C6E;TELUGU DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; 0C6F;TELUGU DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +0C77;TELUGU SIGN SIDDHAM;Po;0;L;;;;;N;;;;; 0C78;TELUGU FRACTION DIGIT ZERO FOR ODD POWERS OF FOUR;No;0;ON;;;;0;N;;;;; 0C79;TELUGU FRACTION DIGIT ONE FOR ODD POWERS OF FOUR;No;0;ON;;;;1;N;;;;; 0C7A;TELUGU FRACTION DIGIT TWO FOR ODD POWERS OF FOUR;No;0;ON;;;;2;N;;;;; @@ -3203,14 +3204,24 @@ 0E81;LAO LETTER KO;Lo;0;L;;;;;N;;;;; 0E82;LAO LETTER KHO SUNG;Lo;0;L;;;;;N;;;;; 0E84;LAO LETTER KHO TAM;Lo;0;L;;;;;N;;;;; +0E86;LAO LETTER PALI GHA;Lo;0;L;;;;;N;;;;; 0E87;LAO LETTER NGO;Lo;0;L;;;;;N;;;;; 0E88;LAO LETTER CO;Lo;0;L;;;;;N;;;;; +0E89;LAO LETTER PALI CHA;Lo;0;L;;;;;N;;;;; 0E8A;LAO LETTER SO TAM;Lo;0;L;;;;;N;;;;; +0E8C;LAO LETTER PALI JHA;Lo;0;L;;;;;N;;;;; 0E8D;LAO LETTER NYO;Lo;0;L;;;;;N;;;;; +0E8E;LAO LETTER PALI NYA;Lo;0;L;;;;;N;;;;; +0E8F;LAO LETTER PALI TTA;Lo;0;L;;;;;N;;;;; +0E90;LAO LETTER PALI TTHA;Lo;0;L;;;;;N;;;;; +0E91;LAO LETTER PALI DDA;Lo;0;L;;;;;N;;;;; +0E92;LAO LETTER PALI DDHA;Lo;0;L;;;;;N;;;;; +0E93;LAO LETTER PALI NNA;Lo;0;L;;;;;N;;;;; 0E94;LAO LETTER DO;Lo;0;L;;;;;N;;;;; 0E95;LAO LETTER TO;Lo;0;L;;;;;N;;;;; 0E96;LAO LETTER THO SUNG;Lo;0;L;;;;;N;;;;; 0E97;LAO LETTER THO TAM;Lo;0;L;;;;;N;;;;; +0E98;LAO LETTER PALI DHA;Lo;0;L;;;;;N;;;;; 0E99;LAO LETTER NO;Lo;0;L;;;;;N;;;;; 0E9A;LAO LETTER BO;Lo;0;L;;;;;N;;;;; 0E9B;LAO LETTER PO;Lo;0;L;;;;;N;;;;; @@ -3218,13 +3229,17 @@ 0E9D;LAO LETTER FO TAM;Lo;0;L;;;;;N;;;;; 0E9E;LAO LETTER PHO TAM;Lo;0;L;;;;;N;;;;; 0E9F;LAO LETTER FO SUNG;Lo;0;L;;;;;N;;;;; +0EA0;LAO LETTER PALI BHA;Lo;0;L;;;;;N;;;;; 0EA1;LAO LETTER MO;Lo;0;L;;;;;N;;;;; 0EA2;LAO LETTER YO;Lo;0;L;;;;;N;;;;; 0EA3;LAO LETTER LO LING;Lo;0;L;;;;;N;;;;; 0EA5;LAO LETTER LO LOOT;Lo;0;L;;;;;N;;;;; 0EA7;LAO LETTER WO;Lo;0;L;;;;;N;;;;; +0EA8;LAO LETTER SANSKRIT SHA;Lo;0;L;;;;;N;;;;; +0EA9;LAO LETTER SANSKRIT SSA;Lo;0;L;;;;;N;;;;; 0EAA;LAO LETTER SO SUNG;Lo;0;L;;;;;N;;;;; 0EAB;LAO LETTER HO SUNG;Lo;0;L;;;;;N;;;;; +0EAC;LAO LETTER PALI LLA;Lo;0;L;;;;;N;;;;; 0EAD;LAO LETTER O;Lo;0;L;;;;;N;;;;; 0EAE;LAO LETTER HO TAM;Lo;0;L;;;;;N;;;;; 0EAF;LAO ELLIPSIS;Lo;0;L;;;;;N;;;;; @@ -3238,6 +3253,7 @@ 0EB7;LAO VOWEL SIGN YY;Mn;0;NSM;;;;;N;;;;; 0EB8;LAO VOWEL SIGN U;Mn;118;NSM;;;;;N;;;;; 0EB9;LAO VOWEL SIGN UU;Mn;118;NSM;;;;;N;;;;; +0EBA;LAO SIGN PALI VIRAMA;Mn;9;NSM;;;;;N;;;;; 0EBB;LAO VOWEL SIGN MAI KON;Mn;0;NSM;;;;;N;;;;; 0EBC;LAO SEMIVOWEL SIGN LO;Mn;0;NSM;;;;;N;;;;; 0EBD;LAO SEMIVOWEL SIGN NYO;Lo;0;L;;;;;N;;;;; @@ -5079,7 +5095,7 @@ 166A;CANADIAN SYLLABICS CARRIER TTSEE;Lo;0;L;;;;;N;;;;; 166B;CANADIAN SYLLABICS CARRIER TTSI;Lo;0;L;;;;;N;;;;; 166C;CANADIAN SYLLABICS CARRIER TTSA;Lo;0;L;;;;;N;;;;; -166D;CANADIAN SYLLABICS CHI SIGN;Po;0;L;;;;;N;;;;; +166D;CANADIAN SYLLABICS CHI SIGN;So;0;L;;;;;N;;;;; 166E;CANADIAN SYLLABICS FULL STOP;Po;0;L;;;;;N;;;;; 166F;CANADIAN SYLLABICS QAI;Lo;0;L;;;;;N;;;;; 1670;CANADIAN SYLLABICS NGAI;Lo;0;L;;;;;N;;;;; @@ -6488,14 +6504,15 @@ 1CEF;VEDIC SIGN LONG ANUSVARA;Lo;0;L;;;;;N;;;;; 1CF0;VEDIC SIGN RTHANG LONG ANUSVARA;Lo;0;L;;;;;N;;;;; 1CF1;VEDIC SIGN ANUSVARA UBHAYATO MUKHA;Lo;0;L;;;;;N;;;;; -1CF2;VEDIC SIGN ARDHAVISARGA;Mc;0;L;;;;;N;;;;; -1CF3;VEDIC SIGN ROTATED ARDHAVISARGA;Mc;0;L;;;;;N;;;;; +1CF2;VEDIC SIGN ARDHAVISARGA;Lo;0;L;;;;;N;;;;; +1CF3;VEDIC SIGN ROTATED ARDHAVISARGA;Lo;0;L;;;;;N;;;;; 1CF4;VEDIC TONE CANDRA ABOVE;Mn;230;NSM;;;;;N;;;;; 1CF5;VEDIC SIGN JIHVAMULIYA;Lo;0;L;;;;;N;;;;; 1CF6;VEDIC SIGN UPADHMANIYA;Lo;0;L;;;;;N;;;;; 1CF7;VEDIC SIGN ATIKRAMA;Mc;0;L;;;;;N;;;;; 1CF8;VEDIC TONE RING ABOVE;Mn;230;NSM;;;;;N;;;;; 1CF9;VEDIC TONE DOUBLE RING ABOVE;Mn;230;NSM;;;;;N;;;;; +1CFA;VEDIC SIGN DOUBLE ANUSVARA ANTARGOMUKHA;Lo;0;L;;;;;N;;;;; 1D00;LATIN LETTER SMALL CAPITAL A;Ll;0;L;;;;;N;;;;; 1D01;LATIN LETTER SMALL CAPITAL AE;Ll;0;L;;;;;N;;;;; 1D02;LATIN SMALL LETTER TURNED AE;Ll;0;L;;;;;N;;;;; @@ -6638,7 +6655,7 @@ 1D8B;LATIN SMALL LETTER ESH WITH PALATAL HOOK;Ll;0;L;;;;;N;;;;; 1D8C;LATIN SMALL LETTER V WITH PALATAL HOOK;Ll;0;L;;;;;N;;;;; 1D8D;LATIN SMALL LETTER X WITH PALATAL HOOK;Ll;0;L;;;;;N;;;;; -1D8E;LATIN SMALL LETTER Z WITH PALATAL HOOK;Ll;0;L;;;;;N;;;;; +1D8E;LATIN SMALL LETTER Z WITH PALATAL HOOK;Ll;0;L;;;;;N;;;A7C6;;A7C6 1D8F;LATIN SMALL LETTER A WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;; 1D90;LATIN SMALL LETTER ALPHA WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;; 1D91;LATIN SMALL LETTER D WITH HOOK AND TAIL;Ll;0;L;;;;;N;;;;; @@ -10165,6 +10182,7 @@ 2BC6;BLACK MEDIUM DOWN-POINTING TRIANGLE CENTRED;So;0;ON;;;;;N;;;;; 2BC7;BLACK MEDIUM LEFT-POINTING TRIANGLE CENTRED;So;0;ON;;;;;N;;;;; 2BC8;BLACK MEDIUM RIGHT-POINTING TRIANGLE CENTRED;So;0;ON;;;;;N;;;;; +2BC9;NEPTUNE FORM TWO;So;0;ON;;;;;N;;;;; 2BCA;TOP HALF BLACK CIRCLE;So;0;ON;;;;;N;;;;; 2BCB;BOTTOM HALF BLACK CIRCLE;So;0;ON;;;;;N;;;;; 2BCC;LIGHT FOUR POINTED BLACK CUSP;So;0;ON;;;;;N;;;;; @@ -10218,6 +10236,7 @@ 2BFC;DOUBLED SYMBOL;So;0;ON;;;;;N;;;;; 2BFD;PASSED SYMBOL;So;0;ON;;;;;N;;;;; 2BFE;REVERSED RIGHT ANGLE;So;0;ON;;;;;Y;;;;; +2BFF;HELLSCHREIBER PAUSE SYMBOL;So;0;ON;;;;;N;;;;; 2C00;GLAGOLITIC CAPITAL LETTER AZU;Lu;0;L;;;;;N;;;;2C30; 2C01;GLAGOLITIC CAPITAL LETTER BUKY;Lu;0;L;;;;;N;;;;2C31; 2C02;GLAGOLITIC CAPITAL LETTER VEDE;Lu;0;L;;;;;N;;;;2C32; @@ -10756,6 +10775,7 @@ 2E4C;MEDIEVAL COMMA;Po;0;ON;;;;;N;;;;; 2E4D;PARAGRAPHUS MARK;Po;0;ON;;;;;N;;;;; 2E4E;PUNCTUS ELEVATUS MARK;Po;0;ON;;;;;N;;;;; +2E4F;CORNISH VERSE DIVIDER;Po;0;ON;;;;;N;;;;; 2E80;CJK RADICAL REPEAT;So;0;ON;;;;;N;;;;; 2E81;CJK RADICAL CLIFF;So;0;ON;;;;;N;;;;; 2E82;CJK RADICAL SECOND ONE;So;0;ON;;;;;N;;;;; @@ -11836,6 +11856,7 @@ 32FC;CIRCLED KATAKANA WI;So;0;L;<circle> 30F0;;;;N;;;;; 32FD;CIRCLED KATAKANA WE;So;0;L;<circle> 30F1;;;;N;;;;; 32FE;CIRCLED KATAKANA WO;So;0;L;<circle> 30F2;;;;N;;;;; +32FF;SQUARE ERA NAME REIWA;So;0;L;<square> 4EE4 548C;;;;N;;;;; 3300;SQUARE APAATO;So;0;L;<square> 30A2 30D1 30FC 30C8;;;;N;SQUARED APAATO;;;; 3301;SQUARE ARUHUA;So;0;L;<square> 30A2 30EB 30D5 30A1;;;;N;SQUARED ARUHUA;;;; 3302;SQUARE ANPEA;So;0;L;<square> 30A2 30F3 30DA 30A2;;;;N;SQUARED ANPEA;;;; @@ -14060,7 +14081,7 @@ A790;LATIN CAPITAL LETTER N WITH DESCENDER;Lu;0;L;;;;;N;;;;A791; A791;LATIN SMALL LETTER N WITH DESCENDER;Ll;0;L;;;;;N;;;A790;;A790 A792;LATIN CAPITAL LETTER C WITH BAR;Lu;0;L;;;;;N;;;;A793; A793;LATIN SMALL LETTER C WITH BAR;Ll;0;L;;;;;N;;;A792;;A792 -A794;LATIN SMALL LETTER C WITH PALATAL HOOK;Ll;0;L;;;;;N;;;;; +A794;LATIN SMALL LETTER C WITH PALATAL HOOK;Ll;0;L;;;;;N;;;A7C4;;A7C4 A795;LATIN SMALL LETTER H WITH PALATAL HOOK;Ll;0;L;;;;;N;;;;; A796;LATIN CAPITAL LETTER B WITH FLOURISH;Lu;0;L;;;;;N;;;;A797; A797;LATIN SMALL LETTER B WITH FLOURISH;Ll;0;L;;;;;N;;;A796;;A796 @@ -14098,6 +14119,17 @@ A7B6;LATIN CAPITAL LETTER OMEGA;Lu;0;L;;;;;N;;;;A7B7; A7B7;LATIN SMALL LETTER OMEGA;Ll;0;L;;;;;N;;;A7B6;;A7B6 A7B8;LATIN CAPITAL LETTER U WITH STROKE;Lu;0;L;;;;;N;;;;A7B9; A7B9;LATIN SMALL LETTER U WITH STROKE;Ll;0;L;;;;;N;;;A7B8;;A7B8 +A7BA;LATIN CAPITAL LETTER GLOTTAL A;Lu;0;L;;;;;N;;;;A7BB; +A7BB;LATIN SMALL LETTER GLOTTAL A;Ll;0;L;;;;;N;;;A7BA;;A7BA +A7BC;LATIN CAPITAL LETTER GLOTTAL I;Lu;0;L;;;;;N;;;;A7BD; +A7BD;LATIN SMALL LETTER GLOTTAL I;Ll;0;L;;;;;N;;;A7BC;;A7BC +A7BE;LATIN CAPITAL LETTER GLOTTAL U;Lu;0;L;;;;;N;;;;A7BF; +A7BF;LATIN SMALL LETTER GLOTTAL U;Ll;0;L;;;;;N;;;A7BE;;A7BE +A7C2;LATIN CAPITAL LETTER ANGLICANA W;Lu;0;L;;;;;N;;;;A7C3; +A7C3;LATIN SMALL LETTER ANGLICANA W;Ll;0;L;;;;;N;;;A7C2;;A7C2 +A7C4;LATIN CAPITAL LETTER C WITH PALATAL HOOK;Lu;0;L;;;;;N;;;;A794; +A7C5;LATIN CAPITAL LETTER S WITH HOOK;Lu;0;L;;;;;N;;;;0282; +A7C6;LATIN CAPITAL LETTER Z WITH PALATAL HOOK;Lu;0;L;;;;;N;;;;1D8E; A7F7;LATIN EPIGRAPHIC LETTER SIDEWAYS I;Lo;0;L;;;;;N;;;;; A7F8;MODIFIER LETTER CAPITAL H WITH STROKE;Lm;0;L;<super> 0126;;;;N;;;;; A7F9;MODIFIER LETTER SMALL LIGATURE OE;Lm;0;L;<super> 0153;;;;N;;;;; @@ -14506,7 +14538,7 @@ A9B9;JAVANESE VOWEL SIGN SUKU MENDUT;Mn;0;NSM;;;;;N;;;;; A9BA;JAVANESE VOWEL SIGN TALING;Mc;0;L;;;;;N;;;;; A9BB;JAVANESE VOWEL SIGN DIRGA MURE;Mc;0;L;;;;;N;;;;; A9BC;JAVANESE VOWEL SIGN PEPET;Mn;0;NSM;;;;;N;;;;; -A9BD;JAVANESE CONSONANT SIGN KERET;Mc;0;L;;;;;N;;;;; +A9BD;JAVANESE CONSONANT SIGN KERET;Mn;0;NSM;;;;;N;;;;; A9BE;JAVANESE CONSONANT SIGN PENGKAL;Mc;0;L;;;;;N;;;;; A9BF;JAVANESE CONSONANT SIGN CAKRA;Mc;0;L;;;;;N;;;;; A9C0;JAVANESE PANGKON;Mc;9;L;;;;;N;;;;; @@ -14863,6 +14895,8 @@ AB62;LATIN SMALL LETTER OPEN OE;Ll;0;L;;;;;N;;;;; AB63;LATIN SMALL LETTER UO;Ll;0;L;;;;;N;;;;; AB64;LATIN SMALL LETTER INVERTED ALPHA;Ll;0;L;;;;;N;;;;; AB65;GREEK LETTER SMALL CAPITAL OMEGA;Ll;0;L;;;;;N;;;;; +AB66;LATIN SMALL LETTER DZ DIGRAPH WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;; +AB67;LATIN SMALL LETTER TS DIGRAPH WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;; AB70;CHEROKEE SMALL LETTER A;Ll;0;L;;;;;N;;;13A0;;13A0 AB71;CHEROKEE SMALL LETTER E;Ll;0;L;;;;;N;;;13A1;;13A1 AB72;CHEROKEE SMALL LETTER I;Ll;0;L;;;;;N;;;13A2;;13A2 @@ -19105,6 +19139,29 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 10F57;SOGDIAN PUNCTUATION CIRCLE WITH DOT;Po;0;AL;;;;;N;;;;; 10F58;SOGDIAN PUNCTUATION TWO CIRCLES WITH DOTS;Po;0;AL;;;;;N;;;;; 10F59;SOGDIAN PUNCTUATION HALF CIRCLE WITH DOT;Po;0;AL;;;;;N;;;;; +10FE0;ELYMAIC LETTER ALEPH;Lo;0;R;;;;;N;;;;; +10FE1;ELYMAIC LETTER BETH;Lo;0;R;;;;;N;;;;; +10FE2;ELYMAIC LETTER GIMEL;Lo;0;R;;;;;N;;;;; +10FE3;ELYMAIC LETTER DALETH;Lo;0;R;;;;;N;;;;; +10FE4;ELYMAIC LETTER HE;Lo;0;R;;;;;N;;;;; +10FE5;ELYMAIC LETTER WAW;Lo;0;R;;;;;N;;;;; +10FE6;ELYMAIC LETTER ZAYIN;Lo;0;R;;;;;N;;;;; +10FE7;ELYMAIC LETTER HETH;Lo;0;R;;;;;N;;;;; +10FE8;ELYMAIC LETTER TETH;Lo;0;R;;;;;N;;;;; +10FE9;ELYMAIC LETTER YODH;Lo;0;R;;;;;N;;;;; +10FEA;ELYMAIC LETTER KAPH;Lo;0;R;;;;;N;;;;; +10FEB;ELYMAIC LETTER LAMEDH;Lo;0;R;;;;;N;;;;; +10FEC;ELYMAIC LETTER MEM;Lo;0;R;;;;;N;;;;; +10FED;ELYMAIC LETTER NUN;Lo;0;R;;;;;N;;;;; +10FEE;ELYMAIC LETTER SAMEKH;Lo;0;R;;;;;N;;;;; +10FEF;ELYMAIC LETTER AYIN;Lo;0;R;;;;;N;;;;; +10FF0;ELYMAIC LETTER PE;Lo;0;R;;;;;N;;;;; +10FF1;ELYMAIC LETTER SADHE;Lo;0;R;;;;;N;;;;; +10FF2;ELYMAIC LETTER QOPH;Lo;0;R;;;;;N;;;;; +10FF3;ELYMAIC LETTER RESH;Lo;0;R;;;;;N;;;;; +10FF4;ELYMAIC LETTER SHIN;Lo;0;R;;;;;N;;;;; +10FF5;ELYMAIC LETTER TAW;Lo;0;R;;;;;N;;;;; +10FF6;ELYMAIC LIGATURE ZAYIN-YODH;Lo;0;R;;;;;N;;;;; 11000;BRAHMI SIGN CANDRABINDU;Mc;0;L;;;;;N;;;;; 11001;BRAHMI SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;; 11002;BRAHMI SIGN VISARGA;Mc;0;L;;;;;N;;;;; @@ -19887,6 +19944,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1145B;NEWA PLACEHOLDER MARK;Po;0;L;;;;;N;;;;; 1145D;NEWA INSERTION SIGN;Po;0;L;;;;;N;;;;; 1145E;NEWA SANDHI MARK;Mn;230;NSM;;;;;N;;;;; +1145F;NEWA LETTER VEDIC ANUSVARA;Lo;0;L;;;;;N;;;;; 11480;TIRHUTA ANJI;Lo;0;L;;;;;N;;;;; 11481;TIRHUTA LETTER A;Lo;0;L;;;;;N;;;;; 11482;TIRHUTA LETTER AA;Lo;0;L;;;;;N;;;;; @@ -20209,6 +20267,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 116B5;TAKRI VOWEL SIGN AU;Mn;0;NSM;;;;;N;;;;; 116B6;TAKRI SIGN VIRAMA;Mc;9;L;;;;;N;;;;; 116B7;TAKRI SIGN NUKTA;Mn;7;NSM;;;;;N;;;;; +116B8;TAKRI LETTER ARCHAIC KHA;Lo;0;L;;;;;N;;;;; 116C0;TAKRI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; 116C1;TAKRI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; 116C2;TAKRI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; @@ -20421,6 +20480,71 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 118F1;WARANG CITI NUMBER EIGHTY;No;0;L;;;;80;N;;;;; 118F2;WARANG CITI NUMBER NINETY;No;0;L;;;;90;N;;;;; 118FF;WARANG CITI OM;Lo;0;L;;;;;N;;;;; +119A0;NANDINAGARI LETTER A;Lo;0;L;;;;;N;;;;; +119A1;NANDINAGARI LETTER AA;Lo;0;L;;;;;N;;;;; +119A2;NANDINAGARI LETTER I;Lo;0;L;;;;;N;;;;; +119A3;NANDINAGARI LETTER II;Lo;0;L;;;;;N;;;;; +119A4;NANDINAGARI LETTER U;Lo;0;L;;;;;N;;;;; +119A5;NANDINAGARI LETTER UU;Lo;0;L;;;;;N;;;;; +119A6;NANDINAGARI LETTER VOCALIC R;Lo;0;L;;;;;N;;;;; +119A7;NANDINAGARI LETTER VOCALIC RR;Lo;0;L;;;;;N;;;;; +119AA;NANDINAGARI LETTER E;Lo;0;L;;;;;N;;;;; +119AB;NANDINAGARI LETTER AI;Lo;0;L;;;;;N;;;;; +119AC;NANDINAGARI LETTER O;Lo;0;L;;;;;N;;;;; +119AD;NANDINAGARI LETTER AU;Lo;0;L;;;;;N;;;;; +119AE;NANDINAGARI LETTER KA;Lo;0;L;;;;;N;;;;; +119AF;NANDINAGARI LETTER KHA;Lo;0;L;;;;;N;;;;; +119B0;NANDINAGARI LETTER GA;Lo;0;L;;;;;N;;;;; +119B1;NANDINAGARI LETTER GHA;Lo;0;L;;;;;N;;;;; +119B2;NANDINAGARI LETTER NGA;Lo;0;L;;;;;N;;;;; +119B3;NANDINAGARI LETTER CA;Lo;0;L;;;;;N;;;;; +119B4;NANDINAGARI LETTER CHA;Lo;0;L;;;;;N;;;;; +119B5;NANDINAGARI LETTER JA;Lo;0;L;;;;;N;;;;; +119B6;NANDINAGARI LETTER JHA;Lo;0;L;;;;;N;;;;; +119B7;NANDINAGARI LETTER NYA;Lo;0;L;;;;;N;;;;; +119B8;NANDINAGARI LETTER TTA;Lo;0;L;;;;;N;;;;; +119B9;NANDINAGARI LETTER TTHA;Lo;0;L;;;;;N;;;;; +119BA;NANDINAGARI LETTER DDA;Lo;0;L;;;;;N;;;;; +119BB;NANDINAGARI LETTER DDHA;Lo;0;L;;;;;N;;;;; +119BC;NANDINAGARI LETTER NNA;Lo;0;L;;;;;N;;;;; +119BD;NANDINAGARI LETTER TA;Lo;0;L;;;;;N;;;;; +119BE;NANDINAGARI LETTER THA;Lo;0;L;;;;;N;;;;; +119BF;NANDINAGARI LETTER DA;Lo;0;L;;;;;N;;;;; +119C0;NANDINAGARI LETTER DHA;Lo;0;L;;;;;N;;;;; +119C1;NANDINAGARI LETTER NA;Lo;0;L;;;;;N;;;;; +119C2;NANDINAGARI LETTER PA;Lo;0;L;;;;;N;;;;; +119C3;NANDINAGARI LETTER PHA;Lo;0;L;;;;;N;;;;; +119C4;NANDINAGARI LETTER BA;Lo;0;L;;;;;N;;;;; +119C5;NANDINAGARI LETTER BHA;Lo;0;L;;;;;N;;;;; +119C6;NANDINAGARI LETTER MA;Lo;0;L;;;;;N;;;;; +119C7;NANDINAGARI LETTER YA;Lo;0;L;;;;;N;;;;; +119C8;NANDINAGARI LETTER RA;Lo;0;L;;;;;N;;;;; +119C9;NANDINAGARI LETTER LA;Lo;0;L;;;;;N;;;;; +119CA;NANDINAGARI LETTER VA;Lo;0;L;;;;;N;;;;; +119CB;NANDINAGARI LETTER SHA;Lo;0;L;;;;;N;;;;; +119CC;NANDINAGARI LETTER SSA;Lo;0;L;;;;;N;;;;; +119CD;NANDINAGARI LETTER SA;Lo;0;L;;;;;N;;;;; +119CE;NANDINAGARI LETTER HA;Lo;0;L;;;;;N;;;;; +119CF;NANDINAGARI LETTER LLA;Lo;0;L;;;;;N;;;;; +119D0;NANDINAGARI LETTER RRA;Lo;0;L;;;;;N;;;;; +119D1;NANDINAGARI VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +119D2;NANDINAGARI VOWEL SIGN I;Mc;0;L;;;;;N;;;;; +119D3;NANDINAGARI VOWEL SIGN II;Mc;0;L;;;;;N;;;;; +119D4;NANDINAGARI VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +119D5;NANDINAGARI VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;; +119D6;NANDINAGARI VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;; +119D7;NANDINAGARI VOWEL SIGN VOCALIC RR;Mn;0;NSM;;;;;N;;;;; +119DA;NANDINAGARI VOWEL SIGN E;Mn;0;NSM;;;;;N;;;;; +119DB;NANDINAGARI VOWEL SIGN AI;Mn;0;NSM;;;;;N;;;;; +119DC;NANDINAGARI VOWEL SIGN O;Mc;0;L;;;;;N;;;;; +119DD;NANDINAGARI VOWEL SIGN AU;Mc;0;L;;;;;N;;;;; +119DE;NANDINAGARI SIGN ANUSVARA;Mc;0;L;;;;;N;;;;; +119DF;NANDINAGARI SIGN VISARGA;Mc;0;L;;;;;N;;;;; +119E0;NANDINAGARI SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;; +119E1;NANDINAGARI SIGN AVAGRAHA;Lo;0;L;;;;;N;;;;; +119E2;NANDINAGARI SIGN SIDDHAM;Po;0;L;;;;;N;;;;; +119E3;NANDINAGARI HEADSTROKE;Lo;0;L;;;;;N;;;;; +119E4;NANDINAGARI VOWEL SIGN PRISHTHAMATRA E;Mc;0;L;;;;;N;;;;; 11A00;ZANABAZAR SQUARE LETTER A;Lo;0;L;;;;;N;;;;; 11A01;ZANABAZAR SQUARE VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; 11A02;ZANABAZAR SQUARE VOWEL SIGN UE;Mn;0;NSM;;;;;N;;;;; @@ -20545,6 +20669,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11A81;SOYOMBO LETTER SA;Lo;0;L;;;;;N;;;;; 11A82;SOYOMBO LETTER HA;Lo;0;L;;;;;N;;;;; 11A83;SOYOMBO LETTER KSSA;Lo;0;L;;;;;N;;;;; +11A84;SOYOMBO SIGN JIHVAMULIYA;Lo;0;L;;;;;N;;;;; +11A85;SOYOMBO SIGN UPADHMANIYA;Lo;0;L;;;;;N;;;;; 11A86;SOYOMBO CLUSTER-INITIAL LETTER RA;Lo;0;L;;;;;N;;;;; 11A87;SOYOMBO CLUSTER-INITIAL LETTER LA;Lo;0;L;;;;;N;;;;; 11A88;SOYOMBO CLUSTER-INITIAL LETTER SHA;Lo;0;L;;;;;N;;;;; @@ -20959,6 +21085,57 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11EF6;MAKASAR VOWEL SIGN O;Mc;0;L;;;;;N;;;;; 11EF7;MAKASAR PASSIMBANG;Po;0;L;;;;;N;;;;; 11EF8;MAKASAR END OF SECTION;Po;0;L;;;;;N;;;;; +11FC0;TAMIL FRACTION ONE THREE-HUNDRED-AND-TWENTIETH;No;0;L;;;;1/320;N;;;;; +11FC1;TAMIL FRACTION ONE ONE-HUNDRED-AND-SIXTIETH;No;0;L;;;;1/160;N;;;;; +11FC2;TAMIL FRACTION ONE EIGHTIETH;No;0;L;;;;1/80;N;;;;; +11FC3;TAMIL FRACTION ONE SIXTY-FOURTH;No;0;L;;;;1/64;N;;;;; +11FC4;TAMIL FRACTION ONE FORTIETH;No;0;L;;;;1/40;N;;;;; +11FC5;TAMIL FRACTION ONE THIRTY-SECOND;No;0;L;;;;1/32;N;;;;; +11FC6;TAMIL FRACTION THREE EIGHTIETHS;No;0;L;;;;3/80;N;;;;; +11FC7;TAMIL FRACTION THREE SIXTY-FOURTHS;No;0;L;;;;3/64;N;;;;; +11FC8;TAMIL FRACTION ONE TWENTIETH;No;0;L;;;;1/20;N;;;;; +11FC9;TAMIL FRACTION ONE SIXTEENTH-1;No;0;L;;;;1/16;N;;;;; +11FCA;TAMIL FRACTION ONE SIXTEENTH-2;No;0;L;;;;1/16;N;;;;; +11FCB;TAMIL FRACTION ONE TENTH;No;0;L;;;;1/10;N;;;;; +11FCC;TAMIL FRACTION ONE EIGHTH;No;0;L;;;;1/8;N;;;;; +11FCD;TAMIL FRACTION THREE TWENTIETHS;No;0;L;;;;3/20;N;;;;; +11FCE;TAMIL FRACTION THREE SIXTEENTHS;No;0;L;;;;3/16;N;;;;; +11FCF;TAMIL FRACTION ONE FIFTH;No;0;L;;;;1/5;N;;;;; +11FD0;TAMIL FRACTION ONE QUARTER;No;0;L;;;;1/4;N;;;;; +11FD1;TAMIL FRACTION ONE HALF-1;No;0;L;;;;1/2;N;;;;; +11FD2;TAMIL FRACTION ONE HALF-2;No;0;L;;;;1/2;N;;;;; +11FD3;TAMIL FRACTION THREE QUARTERS;No;0;L;;;;3/4;N;;;;; +11FD4;TAMIL FRACTION DOWNSCALING FACTOR KIIZH;No;0;L;;;;1/320;N;;;;; +11FD5;TAMIL SIGN NEL;So;0;ON;;;;;N;;;;; +11FD6;TAMIL SIGN CEVITU;So;0;ON;;;;;N;;;;; +11FD7;TAMIL SIGN AAZHAAKKU;So;0;ON;;;;;N;;;;; +11FD8;TAMIL SIGN UZHAKKU;So;0;ON;;;;;N;;;;; +11FD9;TAMIL SIGN MUUVUZHAKKU;So;0;ON;;;;;N;;;;; +11FDA;TAMIL SIGN KURUNI;So;0;ON;;;;;N;;;;; +11FDB;TAMIL SIGN PATHAKKU;So;0;ON;;;;;N;;;;; +11FDC;TAMIL SIGN MUKKURUNI;So;0;ON;;;;;N;;;;; +11FDD;TAMIL SIGN KAACU;Sc;0;ET;;;;;N;;;;; +11FDE;TAMIL SIGN PANAM;Sc;0;ET;;;;;N;;;;; +11FDF;TAMIL SIGN PON;Sc;0;ET;;;;;N;;;;; +11FE0;TAMIL SIGN VARAAKAN;Sc;0;ET;;;;;N;;;;; +11FE1;TAMIL SIGN PAARAM;So;0;ON;;;;;N;;;;; +11FE2;TAMIL SIGN KUZHI;So;0;ON;;;;;N;;;;; +11FE3;TAMIL SIGN VELI;So;0;ON;;;;;N;;;;; +11FE4;TAMIL WET CULTIVATION SIGN;So;0;ON;;;;;N;;;;; +11FE5;TAMIL DRY CULTIVATION SIGN;So;0;ON;;;;;N;;;;; +11FE6;TAMIL LAND SIGN;So;0;ON;;;;;N;;;;; +11FE7;TAMIL SALT PAN SIGN;So;0;ON;;;;;N;;;;; +11FE8;TAMIL TRADITIONAL CREDIT SIGN;So;0;ON;;;;;N;;;;; +11FE9;TAMIL TRADITIONAL NUMBER SIGN;So;0;ON;;;;;N;;;;; +11FEA;TAMIL CURRENT SIGN;So;0;ON;;;;;N;;;;; +11FEB;TAMIL AND ODD SIGN;So;0;ON;;;;;N;;;;; +11FEC;TAMIL SPENT SIGN;So;0;ON;;;;;N;;;;; +11FED;TAMIL TOTAL SIGN;So;0;ON;;;;;N;;;;; +11FEE;TAMIL IN POSSESSION SIGN;So;0;ON;;;;;N;;;;; +11FEF;TAMIL STARTING FROM SIGN;So;0;ON;;;;;N;;;;; +11FF0;TAMIL SIGN MUTHALIYA;So;0;ON;;;;;N;;;;; +11FF1;TAMIL SIGN VAKAIYARAA;So;0;ON;;;;;N;;;;; +11FFF;TAMIL PUNCTUATION END OF TEXT;Po;0;L;;;;;N;;;;; 12000;CUNEIFORM SIGN A;Lo;0;L;;;;;N;;;;; 12001;CUNEIFORM SIGN A TIMES A;Lo;0;L;;;;;N;;;;; 12002;CUNEIFORM SIGN A TIMES BAD;Lo;0;L;;;;;N;;;;; @@ -23264,6 +23441,15 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1342C;EGYPTIAN HIEROGLYPH AA030;Lo;0;L;;;;;N;;;;; 1342D;EGYPTIAN HIEROGLYPH AA031;Lo;0;L;;;;;N;;;;; 1342E;EGYPTIAN HIEROGLYPH AA032;Lo;0;L;;;;;N;;;;; +13430;EGYPTIAN HIEROGLYPH VERTICAL JOINER;Cf;0;L;;;;;N;;;;; +13431;EGYPTIAN HIEROGLYPH HORIZONTAL JOINER;Cf;0;L;;;;;N;;;;; +13432;EGYPTIAN HIEROGLYPH INSERT AT TOP START;Cf;0;L;;;;;N;;;;; +13433;EGYPTIAN HIEROGLYPH INSERT AT BOTTOM START;Cf;0;L;;;;;N;;;;; +13434;EGYPTIAN HIEROGLYPH INSERT AT TOP END;Cf;0;L;;;;;N;;;;; +13435;EGYPTIAN HIEROGLYPH INSERT AT BOTTOM END;Cf;0;L;;;;;N;;;;; +13436;EGYPTIAN HIEROGLYPH OVERLAY MIDDLE;Cf;0;L;;;;;N;;;;; +13437;EGYPTIAN HIEROGLYPH BEGIN SEGMENT;Cf;0;L;;;;;N;;;;; +13438;EGYPTIAN HIEROGLYPH END SEGMENT;Cf;0;L;;;;;N;;;;; 14400;ANATOLIAN HIEROGLYPH A001;Lo;0;L;;;;;N;;;;; 14401;ANATOLIAN HIEROGLYPH A002;Lo;0;L;;;;;N;;;;; 14402;ANATOLIAN HIEROGLYPH A003;Lo;0;L;;;;;N;;;;; @@ -24782,6 +24968,13 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 16F42;MIAO LETTER WA;Lo;0;L;;;;;N;;;;; 16F43;MIAO LETTER AH;Lo;0;L;;;;;N;;;;; 16F44;MIAO LETTER HHA;Lo;0;L;;;;;N;;;;; +16F45;MIAO LETTER BRI;Lo;0;L;;;;;N;;;;; +16F46;MIAO LETTER SYI;Lo;0;L;;;;;N;;;;; +16F47;MIAO LETTER DZYI;Lo;0;L;;;;;N;;;;; +16F48;MIAO LETTER TE;Lo;0;L;;;;;N;;;;; +16F49;MIAO LETTER TSE;Lo;0;L;;;;;N;;;;; +16F4A;MIAO LETTER RTE;Lo;0;L;;;;;N;;;;; +16F4F;MIAO SIGN CONSONANT MODIFIER BAR;Mn;0;NSM;;;;;N;;;;; 16F50;MIAO LETTER NASALIZATION;Lo;0;L;;;;;N;;;;; 16F51;MIAO SIGN ASPIRATION;Mc;0;L;;;;;N;;;;; 16F52;MIAO SIGN REFORMED VOICING;Mc;0;L;;;;;N;;;;; @@ -24829,6 +25022,15 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 16F7C;MIAO VOWEL SIGN OU;Mc;0;L;;;;;N;;;;; 16F7D;MIAO VOWEL SIGN N;Mc;0;L;;;;;N;;;;; 16F7E;MIAO VOWEL SIGN NG;Mc;0;L;;;;;N;;;;; +16F7F;MIAO VOWEL SIGN UOG;Mc;0;L;;;;;N;;;;; +16F80;MIAO VOWEL SIGN YUI;Mc;0;L;;;;;N;;;;; +16F81;MIAO VOWEL SIGN OG;Mc;0;L;;;;;N;;;;; +16F82;MIAO VOWEL SIGN OER;Mc;0;L;;;;;N;;;;; +16F83;MIAO VOWEL SIGN VW;Mc;0;L;;;;;N;;;;; +16F84;MIAO VOWEL SIGN IG;Mc;0;L;;;;;N;;;;; +16F85;MIAO VOWEL SIGN EA;Mc;0;L;;;;;N;;;;; +16F86;MIAO VOWEL SIGN IONG;Mc;0;L;;;;;N;;;;; +16F87;MIAO VOWEL SIGN UI;Mc;0;L;;;;;N;;;;; 16F8F;MIAO TONE RIGHT;Mn;0;NSM;;;;;N;;;;; 16F90;MIAO TONE TOP RIGHT;Mn;0;NSM;;;;;N;;;;; 16F91;MIAO TONE ABOVE;Mn;0;NSM;;;;;N;;;;; @@ -24848,8 +25050,10 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 16F9F;MIAO LETTER REFORMED TONE-8;Lm;0;L;;;;;N;;;;; 16FE0;TANGUT ITERATION MARK;Lm;0;L;;;;;N;;;;; 16FE1;NUSHU ITERATION MARK;Lm;0;L;;;;;N;;;;; +16FE2;OLD CHINESE HOOK MARK;Po;0;ON;;;;;N;;;;; +16FE3;OLD CHINESE ITERATION MARK;Lm;0;L;;;;;N;;;;; 17000;<Tangut Ideograph, First>;Lo;0;L;;;;;N;;;;; -187F1;<Tangut Ideograph, Last>;Lo;0;L;;;;;N;;;;; +187F7;<Tangut Ideograph, Last>;Lo;0;L;;;;;N;;;;; 18800;TANGUT COMPONENT-001;Lo;0;L;;;;;N;;;;; 18801;TANGUT COMPONENT-002;Lo;0;L;;;;;N;;;;; 18802;TANGUT COMPONENT-003;Lo;0;L;;;;;N;;;;; @@ -25892,6 +26096,13 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1B11C;HENTAIGANA LETTER WO-7;Lo;0;L;;;;;N;;;;; 1B11D;HENTAIGANA LETTER N-MU-MO-1;Lo;0;L;;;;;N;;;;; 1B11E;HENTAIGANA LETTER N-MU-MO-2;Lo;0;L;;;;;N;;;;; +1B150;HIRAGANA LETTER SMALL WI;Lo;0;L;;;;;N;;;;; +1B151;HIRAGANA LETTER SMALL WE;Lo;0;L;;;;;N;;;;; +1B152;HIRAGANA LETTER SMALL WO;Lo;0;L;;;;;N;;;;; +1B164;KATAKANA LETTER SMALL WI;Lo;0;L;;;;;N;;;;; +1B165;KATAKANA LETTER SMALL WE;Lo;0;L;;;;;N;;;;; +1B166;KATAKANA LETTER SMALL WO;Lo;0;L;;;;;N;;;;; +1B167;KATAKANA LETTER SMALL N;Lo;0;L;;;;;N;;;;; 1B170;NUSHU CHARACTER-1B170;Lo;0;L;;;;;N;;;;; 1B171;NUSHU CHARACTER-1B171;Lo;0;L;;;;;N;;;;; 1B172;NUSHU CHARACTER-1B172;Lo;0;L;;;;;N;;;;; @@ -28820,6 +29031,136 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1E028;COMBINING GLAGOLITIC LETTER BIG YUS;Mn;230;NSM;;;;;N;;;;; 1E029;COMBINING GLAGOLITIC LETTER IOTATED BIG YUS;Mn;230;NSM;;;;;N;;;;; 1E02A;COMBINING GLAGOLITIC LETTER FITA;Mn;230;NSM;;;;;N;;;;; +1E100;NYIAKENG PUACHUE HMONG LETTER MA;Lo;0;L;;;;;N;;;;; +1E101;NYIAKENG PUACHUE HMONG LETTER TSA;Lo;0;L;;;;;N;;;;; +1E102;NYIAKENG PUACHUE HMONG LETTER NTA;Lo;0;L;;;;;N;;;;; +1E103;NYIAKENG PUACHUE HMONG LETTER TA;Lo;0;L;;;;;N;;;;; +1E104;NYIAKENG PUACHUE HMONG LETTER HA;Lo;0;L;;;;;N;;;;; +1E105;NYIAKENG PUACHUE HMONG LETTER NA;Lo;0;L;;;;;N;;;;; +1E106;NYIAKENG PUACHUE HMONG LETTER XA;Lo;0;L;;;;;N;;;;; +1E107;NYIAKENG PUACHUE HMONG LETTER NKA;Lo;0;L;;;;;N;;;;; +1E108;NYIAKENG PUACHUE HMONG LETTER CA;Lo;0;L;;;;;N;;;;; +1E109;NYIAKENG PUACHUE HMONG LETTER LA;Lo;0;L;;;;;N;;;;; +1E10A;NYIAKENG PUACHUE HMONG LETTER SA;Lo;0;L;;;;;N;;;;; +1E10B;NYIAKENG PUACHUE HMONG LETTER ZA;Lo;0;L;;;;;N;;;;; +1E10C;NYIAKENG PUACHUE HMONG LETTER NCA;Lo;0;L;;;;;N;;;;; +1E10D;NYIAKENG PUACHUE HMONG LETTER NTSA;Lo;0;L;;;;;N;;;;; +1E10E;NYIAKENG PUACHUE HMONG LETTER KA;Lo;0;L;;;;;N;;;;; +1E10F;NYIAKENG PUACHUE HMONG LETTER DA;Lo;0;L;;;;;N;;;;; +1E110;NYIAKENG PUACHUE HMONG LETTER NYA;Lo;0;L;;;;;N;;;;; +1E111;NYIAKENG PUACHUE HMONG LETTER NRA;Lo;0;L;;;;;N;;;;; +1E112;NYIAKENG PUACHUE HMONG LETTER VA;Lo;0;L;;;;;N;;;;; +1E113;NYIAKENG PUACHUE HMONG LETTER NTXA;Lo;0;L;;;;;N;;;;; +1E114;NYIAKENG PUACHUE HMONG LETTER TXA;Lo;0;L;;;;;N;;;;; +1E115;NYIAKENG PUACHUE HMONG LETTER FA;Lo;0;L;;;;;N;;;;; +1E116;NYIAKENG PUACHUE HMONG LETTER RA;Lo;0;L;;;;;N;;;;; +1E117;NYIAKENG PUACHUE HMONG LETTER QA;Lo;0;L;;;;;N;;;;; +1E118;NYIAKENG PUACHUE HMONG LETTER YA;Lo;0;L;;;;;N;;;;; +1E119;NYIAKENG PUACHUE HMONG LETTER NQA;Lo;0;L;;;;;N;;;;; +1E11A;NYIAKENG PUACHUE HMONG LETTER PA;Lo;0;L;;;;;N;;;;; +1E11B;NYIAKENG PUACHUE HMONG LETTER XYA;Lo;0;L;;;;;N;;;;; +1E11C;NYIAKENG PUACHUE HMONG LETTER NPA;Lo;0;L;;;;;N;;;;; +1E11D;NYIAKENG PUACHUE HMONG LETTER DLA;Lo;0;L;;;;;N;;;;; +1E11E;NYIAKENG PUACHUE HMONG LETTER NPLA;Lo;0;L;;;;;N;;;;; +1E11F;NYIAKENG PUACHUE HMONG LETTER HAH;Lo;0;L;;;;;N;;;;; +1E120;NYIAKENG PUACHUE HMONG LETTER MLA;Lo;0;L;;;;;N;;;;; +1E121;NYIAKENG PUACHUE HMONG LETTER PLA;Lo;0;L;;;;;N;;;;; +1E122;NYIAKENG PUACHUE HMONG LETTER GA;Lo;0;L;;;;;N;;;;; +1E123;NYIAKENG PUACHUE HMONG LETTER RRA;Lo;0;L;;;;;N;;;;; +1E124;NYIAKENG PUACHUE HMONG LETTER A;Lo;0;L;;;;;N;;;;; +1E125;NYIAKENG PUACHUE HMONG LETTER AA;Lo;0;L;;;;;N;;;;; +1E126;NYIAKENG PUACHUE HMONG LETTER I;Lo;0;L;;;;;N;;;;; +1E127;NYIAKENG PUACHUE HMONG LETTER U;Lo;0;L;;;;;N;;;;; +1E128;NYIAKENG PUACHUE HMONG LETTER O;Lo;0;L;;;;;N;;;;; +1E129;NYIAKENG PUACHUE HMONG LETTER OO;Lo;0;L;;;;;N;;;;; +1E12A;NYIAKENG PUACHUE HMONG LETTER E;Lo;0;L;;;;;N;;;;; +1E12B;NYIAKENG PUACHUE HMONG LETTER EE;Lo;0;L;;;;;N;;;;; +1E12C;NYIAKENG PUACHUE HMONG LETTER W;Lo;0;L;;;;;N;;;;; +1E130;NYIAKENG PUACHUE HMONG TONE-B;Mn;230;NSM;;;;;N;;;;; +1E131;NYIAKENG PUACHUE HMONG TONE-M;Mn;230;NSM;;;;;N;;;;; +1E132;NYIAKENG PUACHUE HMONG TONE-J;Mn;230;NSM;;;;;N;;;;; +1E133;NYIAKENG PUACHUE HMONG TONE-V;Mn;230;NSM;;;;;N;;;;; +1E134;NYIAKENG PUACHUE HMONG TONE-S;Mn;230;NSM;;;;;N;;;;; +1E135;NYIAKENG PUACHUE HMONG TONE-G;Mn;230;NSM;;;;;N;;;;; +1E136;NYIAKENG PUACHUE HMONG TONE-D;Mn;230;NSM;;;;;N;;;;; +1E137;NYIAKENG PUACHUE HMONG SIGN FOR PERSON;Lm;0;L;;;;;N;;;;; +1E138;NYIAKENG PUACHUE HMONG SIGN FOR THING;Lm;0;L;;;;;N;;;;; +1E139;NYIAKENG PUACHUE HMONG SIGN FOR LOCATION;Lm;0;L;;;;;N;;;;; +1E13A;NYIAKENG PUACHUE HMONG SIGN FOR ANIMAL;Lm;0;L;;;;;N;;;;; +1E13B;NYIAKENG PUACHUE HMONG SIGN FOR INVERTEBRATE;Lm;0;L;;;;;N;;;;; +1E13C;NYIAKENG PUACHUE HMONG SIGN XW XW;Lm;0;L;;;;;N;;;;; +1E13D;NYIAKENG PUACHUE HMONG SYLLABLE LENGTHENER;Lm;0;L;;;;;N;;;;; +1E140;NYIAKENG PUACHUE HMONG DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +1E141;NYIAKENG PUACHUE HMONG DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +1E142;NYIAKENG PUACHUE HMONG DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +1E143;NYIAKENG PUACHUE HMONG DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +1E144;NYIAKENG PUACHUE HMONG DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +1E145;NYIAKENG PUACHUE HMONG DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +1E146;NYIAKENG PUACHUE HMONG DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +1E147;NYIAKENG PUACHUE HMONG DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +1E148;NYIAKENG PUACHUE HMONG DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +1E149;NYIAKENG PUACHUE HMONG DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +1E14E;NYIAKENG PUACHUE HMONG LOGOGRAM NYAJ;Lo;0;L;;;;;N;;;;; +1E14F;NYIAKENG PUACHUE HMONG CIRCLED CA;So;0;L;;;;;N;;;;; +1E2C0;WANCHO LETTER AA;Lo;0;L;;;;;N;;;;; +1E2C1;WANCHO LETTER A;Lo;0;L;;;;;N;;;;; +1E2C2;WANCHO LETTER BA;Lo;0;L;;;;;N;;;;; +1E2C3;WANCHO LETTER CA;Lo;0;L;;;;;N;;;;; +1E2C4;WANCHO LETTER DA;Lo;0;L;;;;;N;;;;; +1E2C5;WANCHO LETTER GA;Lo;0;L;;;;;N;;;;; +1E2C6;WANCHO LETTER YA;Lo;0;L;;;;;N;;;;; +1E2C7;WANCHO LETTER PHA;Lo;0;L;;;;;N;;;;; +1E2C8;WANCHO LETTER LA;Lo;0;L;;;;;N;;;;; +1E2C9;WANCHO LETTER NA;Lo;0;L;;;;;N;;;;; +1E2CA;WANCHO LETTER PA;Lo;0;L;;;;;N;;;;; +1E2CB;WANCHO LETTER TA;Lo;0;L;;;;;N;;;;; +1E2CC;WANCHO LETTER THA;Lo;0;L;;;;;N;;;;; +1E2CD;WANCHO LETTER FA;Lo;0;L;;;;;N;;;;; +1E2CE;WANCHO LETTER SA;Lo;0;L;;;;;N;;;;; +1E2CF;WANCHO LETTER SHA;Lo;0;L;;;;;N;;;;; +1E2D0;WANCHO LETTER JA;Lo;0;L;;;;;N;;;;; +1E2D1;WANCHO LETTER ZA;Lo;0;L;;;;;N;;;;; +1E2D2;WANCHO LETTER WA;Lo;0;L;;;;;N;;;;; +1E2D3;WANCHO LETTER VA;Lo;0;L;;;;;N;;;;; +1E2D4;WANCHO LETTER KA;Lo;0;L;;;;;N;;;;; +1E2D5;WANCHO LETTER O;Lo;0;L;;;;;N;;;;; +1E2D6;WANCHO LETTER AU;Lo;0;L;;;;;N;;;;; +1E2D7;WANCHO LETTER RA;Lo;0;L;;;;;N;;;;; +1E2D8;WANCHO LETTER MA;Lo;0;L;;;;;N;;;;; +1E2D9;WANCHO LETTER KHA;Lo;0;L;;;;;N;;;;; +1E2DA;WANCHO LETTER HA;Lo;0;L;;;;;N;;;;; +1E2DB;WANCHO LETTER E;Lo;0;L;;;;;N;;;;; +1E2DC;WANCHO LETTER I;Lo;0;L;;;;;N;;;;; +1E2DD;WANCHO LETTER NGA;Lo;0;L;;;;;N;;;;; +1E2DE;WANCHO LETTER U;Lo;0;L;;;;;N;;;;; +1E2DF;WANCHO LETTER LLHA;Lo;0;L;;;;;N;;;;; +1E2E0;WANCHO LETTER TSA;Lo;0;L;;;;;N;;;;; +1E2E1;WANCHO LETTER TRA;Lo;0;L;;;;;N;;;;; +1E2E2;WANCHO LETTER ONG;Lo;0;L;;;;;N;;;;; +1E2E3;WANCHO LETTER AANG;Lo;0;L;;;;;N;;;;; +1E2E4;WANCHO LETTER ANG;Lo;0;L;;;;;N;;;;; +1E2E5;WANCHO LETTER ING;Lo;0;L;;;;;N;;;;; +1E2E6;WANCHO LETTER ON;Lo;0;L;;;;;N;;;;; +1E2E7;WANCHO LETTER EN;Lo;0;L;;;;;N;;;;; +1E2E8;WANCHO LETTER AAN;Lo;0;L;;;;;N;;;;; +1E2E9;WANCHO LETTER NYA;Lo;0;L;;;;;N;;;;; +1E2EA;WANCHO LETTER UEN;Lo;0;L;;;;;N;;;;; +1E2EB;WANCHO LETTER YIH;Lo;0;L;;;;;N;;;;; +1E2EC;WANCHO TONE TUP;Mn;230;NSM;;;;;N;;;;; +1E2ED;WANCHO TONE TUPNI;Mn;230;NSM;;;;;N;;;;; +1E2EE;WANCHO TONE KOI;Mn;230;NSM;;;;;N;;;;; +1E2EF;WANCHO TONE KOINI;Mn;230;NSM;;;;;N;;;;; +1E2F0;WANCHO DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +1E2F1;WANCHO DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +1E2F2;WANCHO DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +1E2F3;WANCHO DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +1E2F4;WANCHO DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +1E2F5;WANCHO DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +1E2F6;WANCHO DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +1E2F7;WANCHO DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +1E2F8;WANCHO DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +1E2F9;WANCHO DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +1E2FF;WANCHO NGUN SIGN;Sc;0;ET;;;;;N;;;;; 1E800;MENDE KIKAKUI SYLLABLE M001 KI;Lo;0;R;;;;;N;;;;; 1E801;MENDE KIKAKUI SYLLABLE M002 KA;Lo;0;R;;;;;N;;;;; 1E802;MENDE KIKAKUI SYLLABLE M003 KU;Lo;0;R;;;;;N;;;;; @@ -29108,6 +29449,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1E948;ADLAM CONSONANT MODIFIER;Mn;230;NSM;;;;;N;;;;; 1E949;ADLAM GEMINATE CONSONANT MODIFIER;Mn;230;NSM;;;;;N;;;;; 1E94A;ADLAM NUKTA;Mn;7;NSM;;;;;N;;;;; +1E94B;ADLAM NASALIZATION MARK;Lm;0;R;;;;;N;;;;; 1E950;ADLAM DIGIT ZERO;Nd;0;R;;0;0;0;N;;;;; 1E951;ADLAM DIGIT ONE;Nd;0;R;;1;1;1;N;;;;; 1E952;ADLAM DIGIT TWO;Nd;0;R;;2;2;2;N;;;;; @@ -29188,6 +29530,67 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1ECB2;INDIC SIYAQ NUMBER ALTERNATE TWO;No;0;AL;;;;2;N;;;;; 1ECB3;INDIC SIYAQ NUMBER ALTERNATE TEN THOUSAND;No;0;AL;;;;10000;N;;;;; 1ECB4;INDIC SIYAQ ALTERNATE LAKH MARK;No;0;AL;;;;100000;N;;;;; +1ED01;OTTOMAN SIYAQ NUMBER ONE;No;0;AL;;;;1;N;;;;; +1ED02;OTTOMAN SIYAQ NUMBER TWO;No;0;AL;;;;2;N;;;;; +1ED03;OTTOMAN SIYAQ NUMBER THREE;No;0;AL;;;;3;N;;;;; +1ED04;OTTOMAN SIYAQ NUMBER FOUR;No;0;AL;;;;4;N;;;;; +1ED05;OTTOMAN SIYAQ NUMBER FIVE;No;0;AL;;;;5;N;;;;; +1ED06;OTTOMAN SIYAQ NUMBER SIX;No;0;AL;;;;6;N;;;;; +1ED07;OTTOMAN SIYAQ NUMBER SEVEN;No;0;AL;;;;7;N;;;;; +1ED08;OTTOMAN SIYAQ NUMBER EIGHT;No;0;AL;;;;8;N;;;;; +1ED09;OTTOMAN SIYAQ NUMBER NINE;No;0;AL;;;;9;N;;;;; +1ED0A;OTTOMAN SIYAQ NUMBER TEN;No;0;AL;;;;10;N;;;;; +1ED0B;OTTOMAN SIYAQ NUMBER TWENTY;No;0;AL;;;;20;N;;;;; +1ED0C;OTTOMAN SIYAQ NUMBER THIRTY;No;0;AL;;;;30;N;;;;; +1ED0D;OTTOMAN SIYAQ NUMBER FORTY;No;0;AL;;;;40;N;;;;; +1ED0E;OTTOMAN SIYAQ NUMBER FIFTY;No;0;AL;;;;50;N;;;;; +1ED0F;OTTOMAN SIYAQ NUMBER SIXTY;No;0;AL;;;;60;N;;;;; +1ED10;OTTOMAN SIYAQ NUMBER SEVENTY;No;0;AL;;;;70;N;;;;; +1ED11;OTTOMAN SIYAQ NUMBER EIGHTY;No;0;AL;;;;80;N;;;;; +1ED12;OTTOMAN SIYAQ NUMBER NINETY;No;0;AL;;;;90;N;;;;; +1ED13;OTTOMAN SIYAQ NUMBER ONE HUNDRED;No;0;AL;;;;100;N;;;;; +1ED14;OTTOMAN SIYAQ NUMBER TWO HUNDRED;No;0;AL;;;;200;N;;;;; +1ED15;OTTOMAN SIYAQ NUMBER THREE HUNDRED;No;0;AL;;;;300;N;;;;; +1ED16;OTTOMAN SIYAQ NUMBER FOUR HUNDRED;No;0;AL;;;;400;N;;;;; +1ED17;OTTOMAN SIYAQ NUMBER FIVE HUNDRED;No;0;AL;;;;500;N;;;;; +1ED18;OTTOMAN SIYAQ NUMBER SIX HUNDRED;No;0;AL;;;;600;N;;;;; +1ED19;OTTOMAN SIYAQ NUMBER SEVEN HUNDRED;No;0;AL;;;;700;N;;;;; +1ED1A;OTTOMAN SIYAQ NUMBER EIGHT HUNDRED;No;0;AL;;;;800;N;;;;; +1ED1B;OTTOMAN SIYAQ NUMBER NINE HUNDRED;No;0;AL;;;;900;N;;;;; +1ED1C;OTTOMAN SIYAQ NUMBER ONE THOUSAND;No;0;AL;;;;1000;N;;;;; +1ED1D;OTTOMAN SIYAQ NUMBER TWO THOUSAND;No;0;AL;;;;2000;N;;;;; +1ED1E;OTTOMAN SIYAQ NUMBER THREE THOUSAND;No;0;AL;;;;3000;N;;;;; +1ED1F;OTTOMAN SIYAQ NUMBER FOUR THOUSAND;No;0;AL;;;;4000;N;;;;; +1ED20;OTTOMAN SIYAQ NUMBER FIVE THOUSAND;No;0;AL;;;;5000;N;;;;; +1ED21;OTTOMAN SIYAQ NUMBER SIX THOUSAND;No;0;AL;;;;6000;N;;;;; +1ED22;OTTOMAN SIYAQ NUMBER SEVEN THOUSAND;No;0;AL;;;;7000;N;;;;; +1ED23;OTTOMAN SIYAQ NUMBER EIGHT THOUSAND;No;0;AL;;;;8000;N;;;;; +1ED24;OTTOMAN SIYAQ NUMBER NINE THOUSAND;No;0;AL;;;;9000;N;;;;; +1ED25;OTTOMAN SIYAQ NUMBER TEN THOUSAND;No;0;AL;;;;10000;N;;;;; +1ED26;OTTOMAN SIYAQ NUMBER TWENTY THOUSAND;No;0;AL;;;;20000;N;;;;; +1ED27;OTTOMAN SIYAQ NUMBER THIRTY THOUSAND;No;0;AL;;;;30000;N;;;;; +1ED28;OTTOMAN SIYAQ NUMBER FORTY THOUSAND;No;0;AL;;;;40000;N;;;;; +1ED29;OTTOMAN SIYAQ NUMBER FIFTY THOUSAND;No;0;AL;;;;50000;N;;;;; +1ED2A;OTTOMAN SIYAQ NUMBER SIXTY THOUSAND;No;0;AL;;;;60000;N;;;;; +1ED2B;OTTOMAN SIYAQ NUMBER SEVENTY THOUSAND;No;0;AL;;;;70000;N;;;;; +1ED2C;OTTOMAN SIYAQ NUMBER EIGHTY THOUSAND;No;0;AL;;;;80000;N;;;;; +1ED2D;OTTOMAN SIYAQ NUMBER NINETY THOUSAND;No;0;AL;;;;90000;N;;;;; +1ED2E;OTTOMAN SIYAQ MARRATAN;So;0;AL;;;;;N;;;;; +1ED2F;OTTOMAN SIYAQ ALTERNATE NUMBER TWO;No;0;AL;;;;2;N;;;;; +1ED30;OTTOMAN SIYAQ ALTERNATE NUMBER THREE;No;0;AL;;;;3;N;;;;; +1ED31;OTTOMAN SIYAQ ALTERNATE NUMBER FOUR;No;0;AL;;;;4;N;;;;; +1ED32;OTTOMAN SIYAQ ALTERNATE NUMBER FIVE;No;0;AL;;;;5;N;;;;; +1ED33;OTTOMAN SIYAQ ALTERNATE NUMBER SIX;No;0;AL;;;;6;N;;;;; +1ED34;OTTOMAN SIYAQ ALTERNATE NUMBER SEVEN;No;0;AL;;;;7;N;;;;; +1ED35;OTTOMAN SIYAQ ALTERNATE NUMBER EIGHT;No;0;AL;;;;8;N;;;;; +1ED36;OTTOMAN SIYAQ ALTERNATE NUMBER NINE;No;0;AL;;;;9;N;;;;; +1ED37;OTTOMAN SIYAQ ALTERNATE NUMBER TEN;No;0;AL;;;;10;N;;;;; +1ED38;OTTOMAN SIYAQ ALTERNATE NUMBER FOUR HUNDRED;No;0;AL;;;;400;N;;;;; +1ED39;OTTOMAN SIYAQ ALTERNATE NUMBER SIX HUNDRED;No;0;AL;;;;600;N;;;;; +1ED3A;OTTOMAN SIYAQ ALTERNATE NUMBER TWO THOUSAND;No;0;AL;;;;2000;N;;;;; +1ED3B;OTTOMAN SIYAQ ALTERNATE NUMBER TEN THOUSAND;No;0;AL;;;;10000;N;;;;; +1ED3C;OTTOMAN SIYAQ FRACTION ONE HALF;No;0;AL;;;;1/2;N;;;;; +1ED3D;OTTOMAN SIYAQ FRACTION ONE SIXTH;No;0;AL;;;;1/6;N;;;;; 1EE00;ARABIC MATHEMATICAL ALEF;Lo;0;AL;<font> 0627;;;;N;;;;; 1EE01;ARABIC MATHEMATICAL BEH;Lo;0;AL;<font> 0628;;;;N;;;;; 1EE02;ARABIC MATHEMATICAL JEEM;Lo;0;AL;<font> 062C;;;;N;;;;; @@ -29662,6 +30065,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F169;NEGATIVE CIRCLED LATIN CAPITAL LETTER Z;So;0;L;;;;;N;;;;; 1F16A;RAISED MC SIGN;So;0;ON;<super> 004D 0043;;;;N;;;;; 1F16B;RAISED MD SIGN;So;0;ON;<super> 004D 0044;;;;N;;;;; +1F16C;RAISED MR SIGN;So;0;ON;<super> 004D 0052;;;;N;;;;; 1F170;NEGATIVE SQUARED LATIN CAPITAL LETTER A;So;0;L;;;;;N;;;;; 1F171;NEGATIVE SQUARED LATIN CAPITAL LETTER B;So;0;L;;;;;N;;;;; 1F172;NEGATIVE SQUARED LATIN CAPITAL LETTER C;So;0;L;;;;;N;;;;; @@ -30794,6 +31198,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F6D2;SHOPPING TROLLEY;So;0;ON;;;;;N;;;;; 1F6D3;STUPA;So;0;ON;;;;;N;;;;; 1F6D4;PAGODA;So;0;ON;;;;;N;;;;; +1F6D5;HINDU TEMPLE;So;0;ON;;;;;N;;;;; 1F6E0;HAMMER AND WRENCH;So;0;ON;;;;;N;;;;; 1F6E1;SHIELD;So;0;ON;;;;;N;;;;; 1F6E2;OIL DRUM;So;0;ON;;;;;N;;;;; @@ -30817,6 +31222,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F6F7;SLED;So;0;ON;;;;;N;;;;; 1F6F8;FLYING SAUCER;So;0;ON;;;;;N;;;;; 1F6F9;SKATEBOARD;So;0;ON;;;;;N;;;;; +1F6FA;AUTO RICKSHAW;So;0;ON;;;;;N;;;;; 1F700;ALCHEMICAL SYMBOL FOR QUINTESSENCE;So;0;ON;;;;;N;;;;; 1F701;ALCHEMICAL SYMBOL FOR AIR;So;0;ON;;;;;N;;;;; 1F702;ALCHEMICAL SYMBOL FOR FIRE;So;0;ON;;;;;N;;;;; @@ -31022,6 +31428,18 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F7D6;NEGATIVE CIRCLED TRIANGLE;So;0;ON;;;;;N;;;;; 1F7D7;CIRCLED SQUARE;So;0;ON;;;;;N;;;;; 1F7D8;NEGATIVE CIRCLED SQUARE;So;0;ON;;;;;N;;;;; +1F7E0;LARGE ORANGE CIRCLE;So;0;ON;;;;;N;;;;; +1F7E1;LARGE YELLOW CIRCLE;So;0;ON;;;;;N;;;;; +1F7E2;LARGE GREEN CIRCLE;So;0;ON;;;;;N;;;;; +1F7E3;LARGE PURPLE CIRCLE;So;0;ON;;;;;N;;;;; +1F7E4;LARGE BROWN CIRCLE;So;0;ON;;;;;N;;;;; +1F7E5;LARGE RED SQUARE;So;0;ON;;;;;N;;;;; +1F7E6;LARGE BLUE SQUARE;So;0;ON;;;;;N;;;;; +1F7E7;LARGE ORANGE SQUARE;So;0;ON;;;;;N;;;;; +1F7E8;LARGE YELLOW SQUARE;So;0;ON;;;;;N;;;;; +1F7E9;LARGE GREEN SQUARE;So;0;ON;;;;;N;;;;; +1F7EA;LARGE PURPLE SQUARE;So;0;ON;;;;;N;;;;; +1F7EB;LARGE BROWN SQUARE;So;0;ON;;;;;N;;;;; 1F800;LEFTWARDS ARROW WITH SMALL TRIANGLE ARROWHEAD;So;0;ON;;;;;N;;;;; 1F801;UPWARDS ARROW WITH SMALL TRIANGLE ARROWHEAD;So;0;ON;;;;;N;;;;; 1F802;RIGHTWARDS ARROW WITH SMALL TRIANGLE ARROWHEAD;So;0;ON;;;;;N;;;;; @@ -31182,6 +31600,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F909;DOWNWARD FACING NOTCHED HOOK;So;0;ON;;;;;N;;;;; 1F90A;DOWNWARD FACING HOOK WITH DOT;So;0;ON;;;;;N;;;;; 1F90B;DOWNWARD FACING NOTCHED HOOK WITH DOT;So;0;ON;;;;;N;;;;; +1F90D;WHITE HEART;So;0;ON;;;;;N;;;;; +1F90E;BROWN HEART;So;0;ON;;;;;N;;;;; +1F90F;PINCHING HAND;So;0;ON;;;;;N;;;;; 1F910;ZIPPER-MOUTH FACE;So;0;ON;;;;;N;;;;; 1F911;MONEY-MOUTH FACE;So;0;ON;;;;;N;;;;; 1F912;FACE WITH THERMOMETER;So;0;ON;;;;;N;;;;; @@ -31229,6 +31650,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F93C;WRESTLERS;So;0;ON;;;;;N;;;;; 1F93D;WATER POLO;So;0;ON;;;;;N;;;;; 1F93E;HANDBALL;So;0;ON;;;;;N;;;;; +1F93F;DIVING MASK;So;0;ON;;;;;N;;;;; 1F940;WILTED FLOWER;So;0;ON;;;;;N;;;;; 1F941;DRUM WITH DRUMSTICKS;So;0;ON;;;;;N;;;;; 1F942;CLINKING GLASSES;So;0;ON;;;;;N;;;;; @@ -31278,11 +31700,13 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F96E;MOON CAKE;So;0;ON;;;;;N;;;;; 1F96F;BAGEL;So;0;ON;;;;;N;;;;; 1F970;SMILING FACE WITH SMILING EYES AND THREE HEARTS;So;0;ON;;;;;N;;;;; +1F971;YAWNING FACE;So;0;ON;;;;;N;;;;; 1F973;FACE WITH PARTY HORN AND PARTY HAT;So;0;ON;;;;;N;;;;; 1F974;FACE WITH UNEVEN EYES AND WAVY MOUTH;So;0;ON;;;;;N;;;;; 1F975;OVERHEATED FACE;So;0;ON;;;;;N;;;;; 1F976;FREEZING FACE;So;0;ON;;;;;N;;;;; 1F97A;FACE WITH PLEADING EYES;So;0;ON;;;;;N;;;;; +1F97B;SARI;So;0;ON;;;;;N;;;;; 1F97C;LAB COAT;So;0;ON;;;;;N;;;;; 1F97D;GOGGLES;So;0;ON;;;;;N;;;;; 1F97E;HIKING BOOT;So;0;ON;;;;;N;;;;; @@ -31322,6 +31746,14 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F9A0;MICROBE;So;0;ON;;;;;N;;;;; 1F9A1;BADGER;So;0;ON;;;;;N;;;;; 1F9A2;SWAN;So;0;ON;;;;;N;;;;; +1F9A5;SLOTH;So;0;ON;;;;;N;;;;; +1F9A6;OTTER;So;0;ON;;;;;N;;;;; +1F9A7;ORANGUTAN;So;0;ON;;;;;N;;;;; +1F9A8;SKUNK;So;0;ON;;;;;N;;;;; +1F9A9;FLAMINGO;So;0;ON;;;;;N;;;;; +1F9AA;OYSTER;So;0;ON;;;;;N;;;;; +1F9AE;GUIDE DOG;So;0;ON;;;;;N;;;;; +1F9AF;PROBING CANE;So;0;ON;;;;;N;;;;; 1F9B0;EMOJI COMPONENT RED HAIR;So;0;ON;;;;;N;;;;; 1F9B1;EMOJI COMPONENT CURLY HAIR;So;0;ON;;;;;N;;;;; 1F9B2;EMOJI COMPONENT BALD;So;0;ON;;;;;N;;;;; @@ -31332,9 +31764,26 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F9B7;TOOTH;So;0;ON;;;;;N;;;;; 1F9B8;SUPERHERO;So;0;ON;;;;;N;;;;; 1F9B9;SUPERVILLAIN;So;0;ON;;;;;N;;;;; +1F9BA;SAFETY VEST;So;0;ON;;;;;N;;;;; +1F9BB;EAR WITH HEARING AID;So;0;ON;;;;;N;;;;; +1F9BC;MOTORIZED WHEELCHAIR;So;0;ON;;;;;N;;;;; +1F9BD;MANUAL WHEELCHAIR;So;0;ON;;;;;N;;;;; +1F9BE;MECHANICAL ARM;So;0;ON;;;;;N;;;;; +1F9BF;MECHANICAL LEG;So;0;ON;;;;;N;;;;; 1F9C0;CHEESE WEDGE;So;0;ON;;;;;N;;;;; 1F9C1;CUPCAKE;So;0;ON;;;;;N;;;;; 1F9C2;SALT SHAKER;So;0;ON;;;;;N;;;;; +1F9C3;BEVERAGE BOX;So;0;ON;;;;;N;;;;; +1F9C4;GARLIC;So;0;ON;;;;;N;;;;; +1F9C5;ONION;So;0;ON;;;;;N;;;;; +1F9C6;FALAFEL;So;0;ON;;;;;N;;;;; +1F9C7;WAFFLE;So;0;ON;;;;;N;;;;; +1F9C8;BUTTER;So;0;ON;;;;;N;;;;; +1F9C9;MATE DRINK;So;0;ON;;;;;N;;;;; +1F9CA;ICE CUBE;So;0;ON;;;;;N;;;;; +1F9CD;STANDING PERSON;So;0;ON;;;;;N;;;;; +1F9CE;KNEELING PERSON;So;0;ON;;;;;N;;;;; +1F9CF;DEAF PERSON;So;0;ON;;;;;N;;;;; 1F9D0;FACE WITH MONOCLE;So;0;ON;;;;;N;;;;; 1F9D1;ADULT;So;0;ON;;;;;N;;;;; 1F9D2;CHILD;So;0;ON;;;;;N;;;;; @@ -31383,6 +31832,90 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F9FD;SPONGE;So;0;ON;;;;;N;;;;; 1F9FE;RECEIPT;So;0;ON;;;;;N;;;;; 1F9FF;NAZAR AMULET;So;0;ON;;;;;N;;;;; +1FA00;NEUTRAL CHESS KING;So;0;ON;;;;;N;;;;; +1FA01;NEUTRAL CHESS QUEEN;So;0;ON;;;;;N;;;;; +1FA02;NEUTRAL CHESS ROOK;So;0;ON;;;;;N;;;;; +1FA03;NEUTRAL CHESS BISHOP;So;0;ON;;;;;N;;;;; +1FA04;NEUTRAL CHESS KNIGHT;So;0;ON;;;;;N;;;;; +1FA05;NEUTRAL CHESS PAWN;So;0;ON;;;;;N;;;;; +1FA06;WHITE CHESS KNIGHT ROTATED FORTY-FIVE DEGREES;So;0;ON;;;;;N;;;;; +1FA07;BLACK CHESS KNIGHT ROTATED FORTY-FIVE DEGREES;So;0;ON;;;;;N;;;;; +1FA08;NEUTRAL CHESS KNIGHT ROTATED FORTY-FIVE DEGREES;So;0;ON;;;;;N;;;;; +1FA09;WHITE CHESS KING ROTATED NINETY DEGREES;So;0;ON;;;;;N;;;;; +1FA0A;WHITE CHESS QUEEN ROTATED NINETY DEGREES;So;0;ON;;;;;N;;;;; +1FA0B;WHITE CHESS ROOK ROTATED NINETY DEGREES;So;0;ON;;;;;N;;;;; +1FA0C;WHITE CHESS BISHOP ROTATED NINETY DEGREES;So;0;ON;;;;;N;;;;; +1FA0D;WHITE CHESS KNIGHT ROTATED NINETY DEGREES;So;0;ON;;;;;N;;;;; +1FA0E;WHITE CHESS PAWN ROTATED NINETY DEGREES;So;0;ON;;;;;N;;;;; +1FA0F;BLACK CHESS KING ROTATED NINETY DEGREES;So;0;ON;;;;;N;;;;; +1FA10;BLACK CHESS QUEEN ROTATED NINETY DEGREES;So;0;ON;;;;;N;;;;; +1FA11;BLACK CHESS ROOK ROTATED NINETY DEGREES;So;0;ON;;;;;N;;;;; +1FA12;BLACK CHESS BISHOP ROTATED NINETY DEGREES;So;0;ON;;;;;N;;;;; +1FA13;BLACK CHESS KNIGHT ROTATED NINETY DEGREES;So;0;ON;;;;;N;;;;; +1FA14;BLACK CHESS PAWN ROTATED NINETY DEGREES;So;0;ON;;;;;N;;;;; +1FA15;NEUTRAL CHESS KING ROTATED NINETY DEGREES;So;0;ON;;;;;N;;;;; +1FA16;NEUTRAL CHESS QUEEN ROTATED NINETY DEGREES;So;0;ON;;;;;N;;;;; +1FA17;NEUTRAL CHESS ROOK ROTATED NINETY DEGREES;So;0;ON;;;;;N;;;;; +1FA18;NEUTRAL CHESS BISHOP ROTATED NINETY DEGREES;So;0;ON;;;;;N;;;;; +1FA19;NEUTRAL CHESS KNIGHT ROTATED NINETY DEGREES;So;0;ON;;;;;N;;;;; +1FA1A;NEUTRAL CHESS PAWN ROTATED NINETY DEGREES;So;0;ON;;;;;N;;;;; +1FA1B;WHITE CHESS KNIGHT ROTATED ONE HUNDRED THIRTY-FIVE DEGREES;So;0;ON;;;;;N;;;;; +1FA1C;BLACK CHESS KNIGHT ROTATED ONE HUNDRED THIRTY-FIVE DEGREES;So;0;ON;;;;;N;;;;; +1FA1D;NEUTRAL CHESS KNIGHT ROTATED ONE HUNDRED THIRTY-FIVE DEGREES;So;0;ON;;;;;N;;;;; +1FA1E;WHITE CHESS TURNED KING;So;0;ON;;;;;N;;;;; +1FA1F;WHITE CHESS TURNED QUEEN;So;0;ON;;;;;N;;;;; +1FA20;WHITE CHESS TURNED ROOK;So;0;ON;;;;;N;;;;; +1FA21;WHITE CHESS TURNED BISHOP;So;0;ON;;;;;N;;;;; +1FA22;WHITE CHESS TURNED KNIGHT;So;0;ON;;;;;N;;;;; +1FA23;WHITE CHESS TURNED PAWN;So;0;ON;;;;;N;;;;; +1FA24;BLACK CHESS TURNED KING;So;0;ON;;;;;N;;;;; +1FA25;BLACK CHESS TURNED QUEEN;So;0;ON;;;;;N;;;;; +1FA26;BLACK CHESS TURNED ROOK;So;0;ON;;;;;N;;;;; +1FA27;BLACK CHESS TURNED BISHOP;So;0;ON;;;;;N;;;;; +1FA28;BLACK CHESS TURNED KNIGHT;So;0;ON;;;;;N;;;;; +1FA29;BLACK CHESS TURNED PAWN;So;0;ON;;;;;N;;;;; +1FA2A;NEUTRAL CHESS TURNED KING;So;0;ON;;;;;N;;;;; +1FA2B;NEUTRAL CHESS TURNED QUEEN;So;0;ON;;;;;N;;;;; +1FA2C;NEUTRAL CHESS TURNED ROOK;So;0;ON;;;;;N;;;;; +1FA2D;NEUTRAL CHESS TURNED BISHOP;So;0;ON;;;;;N;;;;; +1FA2E;NEUTRAL CHESS TURNED KNIGHT;So;0;ON;;;;;N;;;;; +1FA2F;NEUTRAL CHESS TURNED PAWN;So;0;ON;;;;;N;;;;; +1FA30;WHITE CHESS KNIGHT ROTATED TWO HUNDRED TWENTY-FIVE DEGREES;So;0;ON;;;;;N;;;;; +1FA31;BLACK CHESS KNIGHT ROTATED TWO HUNDRED TWENTY-FIVE DEGREES;So;0;ON;;;;;N;;;;; +1FA32;NEUTRAL CHESS KNIGHT ROTATED TWO HUNDRED TWENTY-FIVE DEGREES;So;0;ON;;;;;N;;;;; +1FA33;WHITE CHESS KING ROTATED TWO HUNDRED SEVENTY DEGREES;So;0;ON;;;;;N;;;;; +1FA34;WHITE CHESS QUEEN ROTATED TWO HUNDRED SEVENTY DEGREES;So;0;ON;;;;;N;;;;; +1FA35;WHITE CHESS ROOK ROTATED TWO HUNDRED SEVENTY DEGREES;So;0;ON;;;;;N;;;;; +1FA36;WHITE CHESS BISHOP ROTATED TWO HUNDRED SEVENTY DEGREES;So;0;ON;;;;;N;;;;; +1FA37;WHITE CHESS KNIGHT ROTATED TWO HUNDRED SEVENTY DEGREES;So;0;ON;;;;;N;;;;; +1FA38;WHITE CHESS PAWN ROTATED TWO HUNDRED SEVENTY DEGREES;So;0;ON;;;;;N;;;;; +1FA39;BLACK CHESS KING ROTATED TWO HUNDRED SEVENTY DEGREES;So;0;ON;;;;;N;;;;; +1FA3A;BLACK CHESS QUEEN ROTATED TWO HUNDRED SEVENTY DEGREES;So;0;ON;;;;;N;;;;; +1FA3B;BLACK CHESS ROOK ROTATED TWO HUNDRED SEVENTY DEGREES;So;0;ON;;;;;N;;;;; +1FA3C;BLACK CHESS BISHOP ROTATED TWO HUNDRED SEVENTY DEGREES;So;0;ON;;;;;N;;;;; +1FA3D;BLACK CHESS KNIGHT ROTATED TWO HUNDRED SEVENTY DEGREES;So;0;ON;;;;;N;;;;; +1FA3E;BLACK CHESS PAWN ROTATED TWO HUNDRED SEVENTY DEGREES;So;0;ON;;;;;N;;;;; +1FA3F;NEUTRAL CHESS KING ROTATED TWO HUNDRED SEVENTY DEGREES;So;0;ON;;;;;N;;;;; +1FA40;NEUTRAL CHESS QUEEN ROTATED TWO HUNDRED SEVENTY DEGREES;So;0;ON;;;;;N;;;;; +1FA41;NEUTRAL CHESS ROOK ROTATED TWO HUNDRED SEVENTY DEGREES;So;0;ON;;;;;N;;;;; +1FA42;NEUTRAL CHESS BISHOP ROTATED TWO HUNDRED SEVENTY DEGREES;So;0;ON;;;;;N;;;;; +1FA43;NEUTRAL CHESS KNIGHT ROTATED TWO HUNDRED SEVENTY DEGREES;So;0;ON;;;;;N;;;;; +1FA44;NEUTRAL CHESS PAWN ROTATED TWO HUNDRED SEVENTY DEGREES;So;0;ON;;;;;N;;;;; +1FA45;WHITE CHESS KNIGHT ROTATED THREE HUNDRED FIFTEEN DEGREES;So;0;ON;;;;;N;;;;; +1FA46;BLACK CHESS KNIGHT ROTATED THREE HUNDRED FIFTEEN DEGREES;So;0;ON;;;;;N;;;;; +1FA47;NEUTRAL CHESS KNIGHT ROTATED THREE HUNDRED FIFTEEN DEGREES;So;0;ON;;;;;N;;;;; +1FA48;WHITE CHESS EQUIHOPPER;So;0;ON;;;;;N;;;;; +1FA49;BLACK CHESS EQUIHOPPER;So;0;ON;;;;;N;;;;; +1FA4A;NEUTRAL CHESS EQUIHOPPER;So;0;ON;;;;;N;;;;; +1FA4B;WHITE CHESS EQUIHOPPER ROTATED NINETY DEGREES;So;0;ON;;;;;N;;;;; +1FA4C;BLACK CHESS EQUIHOPPER ROTATED NINETY DEGREES;So;0;ON;;;;;N;;;;; +1FA4D;NEUTRAL CHESS EQUIHOPPER ROTATED NINETY DEGREES;So;0;ON;;;;;N;;;;; +1FA4E;WHITE CHESS KNIGHT-QUEEN;So;0;ON;;;;;N;;;;; +1FA4F;WHITE CHESS KNIGHT-ROOK;So;0;ON;;;;;N;;;;; +1FA50;WHITE CHESS KNIGHT-BISHOP;So;0;ON;;;;;N;;;;; +1FA51;BLACK CHESS KNIGHT-QUEEN;So;0;ON;;;;;N;;;;; +1FA52;BLACK CHESS KNIGHT-ROOK;So;0;ON;;;;;N;;;;; +1FA53;BLACK CHESS KNIGHT-BISHOP;So;0;ON;;;;;N;;;;; 1FA60;XIANGQI RED GENERAL;So;0;ON;;;;;N;;;;; 1FA61;XIANGQI RED MANDARIN;So;0;ON;;;;;N;;;;; 1FA62;XIANGQI RED ELEPHANT;So;0;ON;;;;;N;;;;; @@ -31397,6 +31930,22 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FA6B;XIANGQI BLACK CHARIOT;So;0;ON;;;;;N;;;;; 1FA6C;XIANGQI BLACK CANNON;So;0;ON;;;;;N;;;;; 1FA6D;XIANGQI BLACK SOLDIER;So;0;ON;;;;;N;;;;; +1FA70;BALLET SHOES;So;0;ON;;;;;N;;;;; +1FA71;ONE-PIECE SWIMSUIT;So;0;ON;;;;;N;;;;; +1FA72;BRIEFS;So;0;ON;;;;;N;;;;; +1FA73;SHORTS;So;0;ON;;;;;N;;;;; +1FA78;DROP OF BLOOD;So;0;ON;;;;;N;;;;; +1FA79;ADHESIVE BANDAGE;So;0;ON;;;;;N;;;;; +1FA7A;STETHOSCOPE;So;0;ON;;;;;N;;;;; +1FA80;YO-YO;So;0;ON;;;;;N;;;;; +1FA81;KITE;So;0;ON;;;;;N;;;;; +1FA82;PARACHUTE;So;0;ON;;;;;N;;;;; +1FA90;RINGED PLANET;So;0;ON;;;;;N;;;;; +1FA91;CHAIR;So;0;ON;;;;;N;;;;; +1FA92;RAZOR;So;0;ON;;;;;N;;;;; +1FA93;AXE;So;0;ON;;;;;N;;;;; +1FA94;DIYA LAMP;So;0;ON;;;;;N;;;;; +1FA95;BANJO;So;0;ON;;;;;N;;;;; 20000;<CJK Ideograph Extension B, First>;Lo;0;L;;;;;N;;;;; 2A6D6;<CJK Ideograph Extension B, Last>;Lo;0;L;;;;;N;;;;; 2A700;<CJK Ideograph Extension C, First>;Lo;0;L;;;;;N;;;;; diff --git a/lib/stdlib/uc_spec/emoji-data.txt b/lib/stdlib/uc_spec/emoji-data.txt index 6e66455252..2fb5c3ff68 100644 --- a/lib/stdlib/uc_spec/emoji-data.txt +++ b/lib/stdlib/uc_spec/emoji-data.txt @@ -1,11 +1,11 @@ # emoji-data.txt -# Date: 2018-02-07, 07:55:18 GMT -# © 2018 Unicode®, Inc. +# Date: 2019-01-15, 12:10:05 GMT +# © 2019 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. # For terms of use, see http://www.unicode.org/terms_of_use.html # # Emoji Data for UTS #51 -# Version: 11.0 +# Version: 12.0 # # For documentation and usage, see http://www.unicode.org/reports/tr51 # @@ -45,7 +45,7 @@ 25FB..25FE ; Emoji # 3.2 [4] (◻️..◾) white medium square..black medium-small square 2600..2604 ; Emoji # 1.1 [5] (☀️..☄️) sun..comet 260E ; Emoji # 1.1 [1] (☎️) telephone -2611 ; Emoji # 1.1 [1] (☑️) ballot box with check +2611 ; Emoji # 1.1 [1] (☑️) check box with check 2614..2615 ; Emoji # 4.0 [2] (☔..☕) umbrella with rain drops..hot beverage 2618 ; Emoji # 4.1 [1] (☘️) shamrock 261D ; Emoji # 1.1 [1] (☝️) index pointing up @@ -82,14 +82,14 @@ 26F7..26FA ; Emoji # 5.2 [4] (⛷️..⛺) skier..tent 26FD ; Emoji # 5.2 [1] (⛽) fuel pump 2702 ; Emoji # 1.1 [1] (✂️) scissors -2705 ; Emoji # 6.0 [1] (✅) white heavy check mark +2705 ; Emoji # 6.0 [1] (✅) check mark button 2708..2709 ; Emoji # 1.1 [2] (✈️..✉️) airplane..envelope 270A..270B ; Emoji # 6.0 [2] (✊..✋) raised fist..raised hand 270C..270D ; Emoji # 1.1 [2] (✌️..✍️) victory hand..writing hand 270F ; Emoji # 1.1 [1] (✏️) pencil 2712 ; Emoji # 1.1 [1] (✒️) black nib -2714 ; Emoji # 1.1 [1] (✔️) heavy check mark -2716 ; Emoji # 1.1 [1] (✖️) heavy multiplication x +2714 ; Emoji # 1.1 [1] (✔️) check mark +2716 ; Emoji # 1.1 [1] (✖️) multiplication sign 271D ; Emoji # 1.1 [1] (✝️) latin cross 2721 ; Emoji # 1.1 [1] (✡️) star of David 2728 ; Emoji # 6.0 [1] (✨) sparkles @@ -100,8 +100,8 @@ 274E ; Emoji # 6.0 [1] (❎) cross mark button 2753..2755 ; Emoji # 6.0 [3] (❓..❕) question mark..white exclamation mark 2757 ; Emoji # 5.2 [1] (❗) exclamation mark -2763..2764 ; Emoji # 1.1 [2] (❣️..❤️) heavy heart exclamation..red heart -2795..2797 ; Emoji # 6.0 [3] (➕..➗) heavy plus sign..heavy division sign +2763..2764 ; Emoji # 1.1 [2] (❣️..❤️) heart exclamation..red heart +2795..2797 ; Emoji # 6.0 [3] (➕..➗) plus sign..division sign 27A1 ; Emoji # 1.1 [1] (➡️) right arrow 27B0 ; Emoji # 6.0 [1] (➰) curly loop 27BF ; Emoji # 6.0 [1] (➿) double curly loop @@ -109,7 +109,7 @@ 2B05..2B07 ; Emoji # 4.0 [3] (⬅️..⬇️) left arrow..down arrow 2B1B..2B1C ; Emoji # 5.1 [2] (⬛..⬜) black large square..white large square 2B50 ; Emoji # 5.1 [1] (⭐) star -2B55 ; Emoji # 5.2 [1] (⭕) heavy large circle +2B55 ; Emoji # 5.2 [1] (⭕) hollow red circle 3030 ; Emoji # 1.1 [1] (〰️) wavy dash 303D ; Emoji # 3.2 [1] (〽️) part alternation mark 3297 ; Emoji # 1.1 [1] (㊗️) Japanese “congratulations” button @@ -206,7 +206,7 @@ 1F62E..1F62F ; Emoji # 6.1 [2] (😮..😯) face with open mouth..hushed face 1F630..1F633 ; Emoji # 6.0 [4] (😰..😳) anxious face with sweat..flushed face 1F634 ; Emoji # 6.1 [1] (😴) sleeping face -1F635..1F640 ; Emoji # 6.0 [12] (😵..🙀) dizzy face..weary cat face +1F635..1F640 ; Emoji # 6.0 [12] (😵..🙀) dizzy face..weary cat 1F641..1F642 ; Emoji # 7.0 [2] (🙁..🙂) slightly frowning face..slightly smiling face 1F643..1F644 ; Emoji # 8.0 [2] (🙃..🙄) upside-down face..face with rolling eyes 1F645..1F64F ; Emoji # 6.0 [11] (🙅..🙏) person gesturing NO..folded hands @@ -214,6 +214,7 @@ 1F6CB..1F6CF ; Emoji # 7.0 [5] (🛋️..🛏️) couch and lamp..bed 1F6D0 ; Emoji # 8.0 [1] (🛐) place of worship 1F6D1..1F6D2 ; Emoji # 9.0 [2] (🛑..🛒) stop sign..shopping cart +1F6D5 ; Emoji # 12.0 [1] (🛕) hindu temple 1F6E0..1F6E5 ; Emoji # 7.0 [6] (🛠️..🛥️) hammer and wrench..motor boat 1F6E9 ; Emoji # 7.0 [1] (🛩️) small airplane 1F6EB..1F6EC ; Emoji # 7.0 [2] (🛫..🛬) airplane departure..airplane arrival @@ -222,6 +223,9 @@ 1F6F4..1F6F6 ; Emoji # 9.0 [3] (🛴..🛶) kick scooter..canoe 1F6F7..1F6F8 ; Emoji # 10.0 [2] (🛷..🛸) sled..flying saucer 1F6F9 ; Emoji # 11.0 [1] (🛹) skateboard +1F6FA ; Emoji # 12.0 [1] (🛺) auto rickshaw +1F7E0..1F7EB ; Emoji # 12.0 [12] (🟠..🟫) orange circle..brown square +1F90D..1F90F ; Emoji # 12.0 [3] (🤍..🤏) white heart..pinching hand 1F910..1F918 ; Emoji # 8.0 [9] (🤐..🤘) zipper-mouth face..sign of the horns 1F919..1F91E ; Emoji # 9.0 [6] (🤙..🤞) call me hand..crossed fingers 1F91F ; Emoji # 10.0 [1] (🤟) love-you gesture @@ -231,27 +235,39 @@ 1F931..1F932 ; Emoji # 10.0 [2] (🤱..🤲) breast-feeding..palms up together 1F933..1F93A ; Emoji # 9.0 [8] (🤳..🤺) selfie..person fencing 1F93C..1F93E ; Emoji # 9.0 [3] (🤼..🤾) people wrestling..person playing handball +1F93F ; Emoji # 12.0 [1] (🤿) diving mask 1F940..1F945 ; Emoji # 9.0 [6] (🥀..🥅) wilted flower..goal net 1F947..1F94B ; Emoji # 9.0 [5] (🥇..🥋) 1st place medal..martial arts uniform 1F94C ; Emoji # 10.0 [1] (🥌) curling stone 1F94D..1F94F ; Emoji # 11.0 [3] (🥍..🥏) lacrosse..flying disc 1F950..1F95E ; Emoji # 9.0 [15] (🥐..🥞) croissant..pancakes 1F95F..1F96B ; Emoji # 10.0 [13] (🥟..🥫) dumpling..canned food -1F96C..1F970 ; Emoji # 11.0 [5] (🥬..🥰) leafy green..smiling face with 3 hearts +1F96C..1F970 ; Emoji # 11.0 [5] (🥬..🥰) leafy green..smiling face with hearts +1F971 ; Emoji # 12.0 [1] (🥱) yawning face 1F973..1F976 ; Emoji # 11.0 [4] (🥳..🥶) partying face..cold face 1F97A ; Emoji # 11.0 [1] (🥺) pleading face -1F97C..1F97F ; Emoji # 11.0 [4] (🥼..🥿) lab coat..woman’s flat shoe -1F980..1F984 ; Emoji # 8.0 [5] (🦀..🦄) crab..unicorn face +1F97B ; Emoji # 12.0 [1] (🥻) sari +1F97C..1F97F ; Emoji # 11.0 [4] (🥼..🥿) lab coat..flat shoe +1F980..1F984 ; Emoji # 8.0 [5] (🦀..🦄) crab..unicorn 1F985..1F991 ; Emoji # 9.0 [13] (🦅..🦑) eagle..squid 1F992..1F997 ; Emoji # 10.0 [6] (🦒..🦗) giraffe..cricket 1F998..1F9A2 ; Emoji # 11.0 [11] (🦘..🦢) kangaroo..swan -1F9B0..1F9B9 ; Emoji # 11.0 [10] (🦰..🦹) red-haired..supervillain +1F9A5..1F9AA ; Emoji # 12.0 [6] (🦥..🦪) sloth..oyster +1F9AE..1F9AF ; Emoji # 12.0 [2] (🦮..🦯) guide dog..probing cane +1F9B0..1F9B9 ; Emoji # 11.0 [10] (🦰..🦹) red hair..supervillain +1F9BA..1F9BF ; Emoji # 12.0 [6] (🦺..🦿) safety vest..mechanical leg 1F9C0 ; Emoji # 8.0 [1] (🧀) cheese wedge 1F9C1..1F9C2 ; Emoji # 11.0 [2] (🧁..🧂) cupcake..salt +1F9C3..1F9CA ; Emoji # 12.0 [8] (🧃..🧊) beverage box..ice cube +1F9CD..1F9CF ; Emoji # 12.0 [3] (🧍..🧏) person standing..deaf person 1F9D0..1F9E6 ; Emoji # 10.0 [23] (🧐..🧦) face with monocle..socks 1F9E7..1F9FF ; Emoji # 11.0 [25] (🧧..🧿) red envelope..nazar amulet +1FA70..1FA73 ; Emoji # 12.0 [4] (🩰..🩳) ballet shoes..shorts +1FA78..1FA7A ; Emoji # 12.0 [3] (🩸..🩺) drop of blood..stethoscope +1FA80..1FA82 ; Emoji # 12.0 [3] (🪀..🪂) yo-yo..parachute +1FA90..1FA95 ; Emoji # 12.0 [6] (🪐..🪕) ringed planet..banjo -# Total elements: 1250 +# Total elements: 1311 # ================================================ @@ -278,19 +294,19 @@ 26F5 ; Emoji_Presentation # 5.2 [1] (⛵) sailboat 26FA ; Emoji_Presentation # 5.2 [1] (⛺) tent 26FD ; Emoji_Presentation # 5.2 [1] (⛽) fuel pump -2705 ; Emoji_Presentation # 6.0 [1] (✅) white heavy check mark +2705 ; Emoji_Presentation # 6.0 [1] (✅) check mark button 270A..270B ; Emoji_Presentation # 6.0 [2] (✊..✋) raised fist..raised hand 2728 ; Emoji_Presentation # 6.0 [1] (✨) sparkles 274C ; Emoji_Presentation # 6.0 [1] (❌) cross mark 274E ; Emoji_Presentation # 6.0 [1] (❎) cross mark button 2753..2755 ; Emoji_Presentation # 6.0 [3] (❓..❕) question mark..white exclamation mark 2757 ; Emoji_Presentation # 5.2 [1] (❗) exclamation mark -2795..2797 ; Emoji_Presentation # 6.0 [3] (➕..➗) heavy plus sign..heavy division sign +2795..2797 ; Emoji_Presentation # 6.0 [3] (➕..➗) plus sign..division sign 27B0 ; Emoji_Presentation # 6.0 [1] (➰) curly loop 27BF ; Emoji_Presentation # 6.0 [1] (➿) double curly loop 2B1B..2B1C ; Emoji_Presentation # 5.1 [2] (⬛..⬜) black large square..white large square 2B50 ; Emoji_Presentation # 5.1 [1] (⭐) star -2B55 ; Emoji_Presentation # 5.2 [1] (⭕) heavy large circle +2B55 ; Emoji_Presentation # 5.2 [1] (⭕) hollow red circle 1F004 ; Emoji_Presentation # 5.1 [1] (🀄) mahjong red dragon 1F0CF ; Emoji_Presentation # 6.0 [1] (🃏) joker 1F18E ; Emoji_Presentation # 6.0 [1] (🆎) AB button (blood type) @@ -349,7 +365,7 @@ 1F62E..1F62F ; Emoji_Presentation # 6.1 [2] (😮..😯) face with open mouth..hushed face 1F630..1F633 ; Emoji_Presentation # 6.0 [4] (😰..😳) anxious face with sweat..flushed face 1F634 ; Emoji_Presentation # 6.1 [1] (😴) sleeping face -1F635..1F640 ; Emoji_Presentation # 6.0 [12] (😵..🙀) dizzy face..weary cat face +1F635..1F640 ; Emoji_Presentation # 6.0 [12] (😵..🙀) dizzy face..weary cat 1F641..1F642 ; Emoji_Presentation # 7.0 [2] (🙁..🙂) slightly frowning face..slightly smiling face 1F643..1F644 ; Emoji_Presentation # 8.0 [2] (🙃..🙄) upside-down face..face with rolling eyes 1F645..1F64F ; Emoji_Presentation # 6.0 [11] (🙅..🙏) person gesturing NO..folded hands @@ -357,10 +373,14 @@ 1F6CC ; Emoji_Presentation # 7.0 [1] (🛌) person in bed 1F6D0 ; Emoji_Presentation # 8.0 [1] (🛐) place of worship 1F6D1..1F6D2 ; Emoji_Presentation # 9.0 [2] (🛑..🛒) stop sign..shopping cart +1F6D5 ; Emoji_Presentation # 12.0 [1] (🛕) hindu temple 1F6EB..1F6EC ; Emoji_Presentation # 7.0 [2] (🛫..🛬) airplane departure..airplane arrival 1F6F4..1F6F6 ; Emoji_Presentation # 9.0 [3] (🛴..🛶) kick scooter..canoe 1F6F7..1F6F8 ; Emoji_Presentation # 10.0 [2] (🛷..🛸) sled..flying saucer 1F6F9 ; Emoji_Presentation # 11.0 [1] (🛹) skateboard +1F6FA ; Emoji_Presentation # 12.0 [1] (🛺) auto rickshaw +1F7E0..1F7EB ; Emoji_Presentation # 12.0 [12] (🟠..🟫) orange circle..brown square +1F90D..1F90F ; Emoji_Presentation # 12.0 [3] (🤍..🤏) white heart..pinching hand 1F910..1F918 ; Emoji_Presentation # 8.0 [9] (🤐..🤘) zipper-mouth face..sign of the horns 1F919..1F91E ; Emoji_Presentation # 9.0 [6] (🤙..🤞) call me hand..crossed fingers 1F91F ; Emoji_Presentation # 10.0 [1] (🤟) love-you gesture @@ -370,27 +390,39 @@ 1F931..1F932 ; Emoji_Presentation # 10.0 [2] (🤱..🤲) breast-feeding..palms up together 1F933..1F93A ; Emoji_Presentation # 9.0 [8] (🤳..🤺) selfie..person fencing 1F93C..1F93E ; Emoji_Presentation # 9.0 [3] (🤼..🤾) people wrestling..person playing handball +1F93F ; Emoji_Presentation # 12.0 [1] (🤿) diving mask 1F940..1F945 ; Emoji_Presentation # 9.0 [6] (🥀..🥅) wilted flower..goal net 1F947..1F94B ; Emoji_Presentation # 9.0 [5] (🥇..🥋) 1st place medal..martial arts uniform 1F94C ; Emoji_Presentation # 10.0 [1] (🥌) curling stone 1F94D..1F94F ; Emoji_Presentation # 11.0 [3] (🥍..🥏) lacrosse..flying disc 1F950..1F95E ; Emoji_Presentation # 9.0 [15] (🥐..🥞) croissant..pancakes 1F95F..1F96B ; Emoji_Presentation # 10.0 [13] (🥟..🥫) dumpling..canned food -1F96C..1F970 ; Emoji_Presentation # 11.0 [5] (🥬..🥰) leafy green..smiling face with 3 hearts +1F96C..1F970 ; Emoji_Presentation # 11.0 [5] (🥬..🥰) leafy green..smiling face with hearts +1F971 ; Emoji_Presentation # 12.0 [1] (🥱) yawning face 1F973..1F976 ; Emoji_Presentation # 11.0 [4] (🥳..🥶) partying face..cold face 1F97A ; Emoji_Presentation # 11.0 [1] (🥺) pleading face -1F97C..1F97F ; Emoji_Presentation # 11.0 [4] (🥼..🥿) lab coat..woman’s flat shoe -1F980..1F984 ; Emoji_Presentation # 8.0 [5] (🦀..🦄) crab..unicorn face +1F97B ; Emoji_Presentation # 12.0 [1] (🥻) sari +1F97C..1F97F ; Emoji_Presentation # 11.0 [4] (🥼..🥿) lab coat..flat shoe +1F980..1F984 ; Emoji_Presentation # 8.0 [5] (🦀..🦄) crab..unicorn 1F985..1F991 ; Emoji_Presentation # 9.0 [13] (🦅..🦑) eagle..squid 1F992..1F997 ; Emoji_Presentation # 10.0 [6] (🦒..🦗) giraffe..cricket 1F998..1F9A2 ; Emoji_Presentation # 11.0 [11] (🦘..🦢) kangaroo..swan -1F9B0..1F9B9 ; Emoji_Presentation # 11.0 [10] (🦰..🦹) red-haired..supervillain +1F9A5..1F9AA ; Emoji_Presentation # 12.0 [6] (🦥..🦪) sloth..oyster +1F9AE..1F9AF ; Emoji_Presentation # 12.0 [2] (🦮..🦯) guide dog..probing cane +1F9B0..1F9B9 ; Emoji_Presentation # 11.0 [10] (🦰..🦹) red hair..supervillain +1F9BA..1F9BF ; Emoji_Presentation # 12.0 [6] (🦺..🦿) safety vest..mechanical leg 1F9C0 ; Emoji_Presentation # 8.0 [1] (🧀) cheese wedge 1F9C1..1F9C2 ; Emoji_Presentation # 11.0 [2] (🧁..🧂) cupcake..salt +1F9C3..1F9CA ; Emoji_Presentation # 12.0 [8] (🧃..🧊) beverage box..ice cube +1F9CD..1F9CF ; Emoji_Presentation # 12.0 [3] (🧍..🧏) person standing..deaf person 1F9D0..1F9E6 ; Emoji_Presentation # 10.0 [23] (🧐..🧦) face with monocle..socks 1F9E7..1F9FF ; Emoji_Presentation # 11.0 [25] (🧧..🧿) red envelope..nazar amulet +1FA70..1FA73 ; Emoji_Presentation # 12.0 [4] (🩰..🩳) ballet shoes..shorts +1FA78..1FA7A ; Emoji_Presentation # 12.0 [3] (🩸..🩺) drop of blood..stethoscope +1FA80..1FA82 ; Emoji_Presentation # 12.0 [3] (🪀..🪂) yo-yo..parachute +1FA90..1FA95 ; Emoji_Presentation # 12.0 [6] (🪐..🪕) ringed planet..banjo -# Total elements: 1032 +# Total elements: 1093 # ================================================ @@ -417,12 +449,12 @@ 1F3CB..1F3CC ; Emoji_Modifier_Base # 7.0 [2] (🏋️..🏌️) person lifting weights..person golfing 1F442..1F443 ; Emoji_Modifier_Base # 6.0 [2] (👂..👃) ear..nose 1F446..1F450 ; Emoji_Modifier_Base # 6.0 [11] (👆..👐) backhand index pointing up..open hands -1F466..1F469 ; Emoji_Modifier_Base # 6.0 [4] (👦..👩) boy..woman -1F46E ; Emoji_Modifier_Base # 6.0 [1] (👮) police officer -1F470..1F478 ; Emoji_Modifier_Base # 6.0 [9] (👰..👸) bride with veil..princess +1F466..1F478 ; Emoji_Modifier_Base # 6.0 [19] (👦..👸) boy..princess 1F47C ; Emoji_Modifier_Base # 6.0 [1] (👼) baby angel 1F481..1F483 ; Emoji_Modifier_Base # 6.0 [3] (💁..💃) person tipping hand..woman dancing 1F485..1F487 ; Emoji_Modifier_Base # 6.0 [3] (💅..💇) nail polish..person getting haircut +1F48F ; Emoji_Modifier_Base # 6.0 [1] (💏) kiss +1F491 ; Emoji_Modifier_Base # 6.0 [1] (💑) couple with heart 1F4AA ; Emoji_Modifier_Base # 6.0 [1] (💪) flexed biceps 1F574..1F575 ; Emoji_Modifier_Base # 7.0 [2] (🕴️..🕵️) man in suit levitating..detective 1F57A ; Emoji_Modifier_Base # 9.0 [1] (🕺) man dancing @@ -434,20 +466,22 @@ 1F6B4..1F6B6 ; Emoji_Modifier_Base # 6.0 [3] (🚴..🚶) person biking..person walking 1F6C0 ; Emoji_Modifier_Base # 6.0 [1] (🛀) person taking bath 1F6CC ; Emoji_Modifier_Base # 7.0 [1] (🛌) person in bed +1F90F ; Emoji_Modifier_Base # 12.0 [1] (🤏) pinching hand 1F918 ; Emoji_Modifier_Base # 8.0 [1] (🤘) sign of the horns -1F919..1F91C ; Emoji_Modifier_Base # 9.0 [4] (🤙..🤜) call me hand..right-facing fist -1F91E ; Emoji_Modifier_Base # 9.0 [1] (🤞) crossed fingers +1F919..1F91E ; Emoji_Modifier_Base # 9.0 [6] (🤙..🤞) call me hand..crossed fingers 1F91F ; Emoji_Modifier_Base # 10.0 [1] (🤟) love-you gesture 1F926 ; Emoji_Modifier_Base # 9.0 [1] (🤦) person facepalming 1F930 ; Emoji_Modifier_Base # 9.0 [1] (🤰) pregnant woman 1F931..1F932 ; Emoji_Modifier_Base # 10.0 [2] (🤱..🤲) breast-feeding..palms up together 1F933..1F939 ; Emoji_Modifier_Base # 9.0 [7] (🤳..🤹) selfie..person juggling -1F93D..1F93E ; Emoji_Modifier_Base # 9.0 [2] (🤽..🤾) person playing water polo..person playing handball +1F93C..1F93E ; Emoji_Modifier_Base # 9.0 [3] (🤼..🤾) people wrestling..person playing handball 1F9B5..1F9B6 ; Emoji_Modifier_Base # 11.0 [2] (🦵..🦶) leg..foot 1F9B8..1F9B9 ; Emoji_Modifier_Base # 11.0 [2] (🦸..🦹) superhero..supervillain -1F9D1..1F9DD ; Emoji_Modifier_Base # 10.0 [13] (🧑..🧝) adult..elf +1F9BB ; Emoji_Modifier_Base # 12.0 [1] (🦻) ear with hearing aid +1F9CD..1F9CF ; Emoji_Modifier_Base # 12.0 [3] (🧍..🧏) person standing..deaf person +1F9D1..1F9DD ; Emoji_Modifier_Base # 10.0 [13] (🧑..🧝) person..elf -# Total elements: 106 +# Total elements: 120 # ================================================ @@ -462,7 +496,7 @@ FE0F ; Emoji_Component # 3.2 [1] () VARIATION SELECTOR-16 1F1E6..1F1FF ; Emoji_Component # 6.0 [26] (🇦..🇿) regional indicator symbol letter a..regional indicator symbol letter z 1F3FB..1F3FF ; Emoji_Component # 8.0 [5] (🏻..🏿) light skin tone..dark skin tone -1F9B0..1F9B3 ; Emoji_Component # 11.0 [4] (🦰..🦳) red-haired..white-haired +1F9B0..1F9B3 ; Emoji_Component # 11.0 [4] (🦰..🦳) red hair..white hair E0020..E007F ; Emoji_Component # 3.1 [96] (..) tag space..cancel tag # Total elements: 146 @@ -482,7 +516,7 @@ E0020..E007F ; Emoji_Component # 3.1 [96] (..) tag space..ca 21A9..21AA ; Extended_Pictographic# 1.1 [2] (↩️..↪️) right arrow curving left..left arrow curving right 231A..231B ; Extended_Pictographic# 1.1 [2] (⌚..⌛) watch..hourglass done 2328 ; Extended_Pictographic# 1.1 [1] (⌨️) keyboard -2388 ; Extended_Pictographic# 3.0 [1] (⎈️) HELM SYMBOL +2388 ; Extended_Pictographic# 3.0 [1] (⎈) HELM SYMBOL 23CF ; Extended_Pictographic# 4.0 [1] (⏏️) eject button 23E9..23F3 ; Extended_Pictographic# 6.0 [11] (⏩..⏳) fast-forward button..hourglass not done 23F8..23FA ; Extended_Pictographic# 7.0 [3] (⏸️..⏺️) pause button..record button @@ -491,42 +525,42 @@ E0020..E007F ; Emoji_Component # 3.1 [96] (..) tag space..ca 25B6 ; Extended_Pictographic# 1.1 [1] (▶️) play button 25C0 ; Extended_Pictographic# 1.1 [1] (◀️) reverse button 25FB..25FE ; Extended_Pictographic# 3.2 [4] (◻️..◾) white medium square..black medium-small square -2600..2605 ; Extended_Pictographic# 1.1 [6] (☀️..★️) sun..BLACK STAR -2607..2612 ; Extended_Pictographic# 1.1 [12] (☇️..☒️) LIGHTNING..BALLOT BOX WITH X +2600..2605 ; Extended_Pictographic# 1.1 [6] (☀️..★) sun..BLACK STAR +2607..2612 ; Extended_Pictographic# 1.1 [12] (☇..☒) LIGHTNING..BALLOT BOX WITH X 2614..2615 ; Extended_Pictographic# 4.0 [2] (☔..☕) umbrella with rain drops..hot beverage -2616..2617 ; Extended_Pictographic# 3.2 [2] (☖️..☗️) WHITE SHOGI PIECE..BLACK SHOGI PIECE +2616..2617 ; Extended_Pictographic# 3.2 [2] (☖..☗) WHITE SHOGI PIECE..BLACK SHOGI PIECE 2618 ; Extended_Pictographic# 4.1 [1] (☘️) shamrock -2619 ; Extended_Pictographic# 3.0 [1] (☙️) REVERSED ROTATED FLORAL HEART BULLET -261A..266F ; Extended_Pictographic# 1.1 [86] (☚️..♯️) BLACK LEFT POINTING INDEX..MUSIC SHARP SIGN -2670..2671 ; Extended_Pictographic# 3.0 [2] (♰️..♱️) WEST SYRIAC CROSS..EAST SYRIAC CROSS -2672..267D ; Extended_Pictographic# 3.2 [12] (♲️..♽️) UNIVERSAL RECYCLING SYMBOL..PARTIALLY-RECYCLED PAPER SYMBOL +2619 ; Extended_Pictographic# 3.0 [1] (☙) REVERSED ROTATED FLORAL HEART BULLET +261A..266F ; Extended_Pictographic# 1.1 [86] (☚..♯) BLACK LEFT POINTING INDEX..MUSIC SHARP SIGN +2670..2671 ; Extended_Pictographic# 3.0 [2] (♰..♱) WEST SYRIAC CROSS..EAST SYRIAC CROSS +2672..267D ; Extended_Pictographic# 3.2 [12] (♲..♽) UNIVERSAL RECYCLING SYMBOL..PARTIALLY-RECYCLED PAPER SYMBOL 267E..267F ; Extended_Pictographic# 4.1 [2] (♾️..♿) infinity..wheelchair symbol -2680..2685 ; Extended_Pictographic# 3.2 [6] (⚀️..⚅️) DIE FACE-1..DIE FACE-6 -2690..2691 ; Extended_Pictographic# 4.0 [2] (⚐️..⚑️) WHITE FLAG..BLACK FLAG +2680..2685 ; Extended_Pictographic# 3.2 [6] (⚀..⚅) DIE FACE-1..DIE FACE-6 +2690..2691 ; Extended_Pictographic# 4.0 [2] (⚐..⚑) WHITE FLAG..BLACK FLAG 2692..269C ; Extended_Pictographic# 4.1 [11] (⚒️..⚜️) hammer and pick..fleur-de-lis -269D ; Extended_Pictographic# 5.1 [1] (⚝️) OUTLINED WHITE STAR -269E..269F ; Extended_Pictographic# 5.2 [2] (⚞️..⚟️) THREE LINES CONVERGING RIGHT..THREE LINES CONVERGING LEFT +269D ; Extended_Pictographic# 5.1 [1] (⚝) OUTLINED WHITE STAR +269E..269F ; Extended_Pictographic# 5.2 [2] (⚞..⚟) THREE LINES CONVERGING RIGHT..THREE LINES CONVERGING LEFT 26A0..26A1 ; Extended_Pictographic# 4.0 [2] (⚠️..⚡) warning..high voltage -26A2..26B1 ; Extended_Pictographic# 4.1 [16] (⚢️..⚱️) DOUBLED FEMALE SIGN..funeral urn -26B2 ; Extended_Pictographic# 5.0 [1] (⚲️) NEUTER -26B3..26BC ; Extended_Pictographic# 5.1 [10] (⚳️..⚼️) CERES..SESQUIQUADRATE -26BD..26BF ; Extended_Pictographic# 5.2 [3] (⚽..⚿️) soccer ball..SQUARED KEY -26C0..26C3 ; Extended_Pictographic# 5.1 [4] (⛀️..⛃️) WHITE DRAUGHTS MAN..BLACK DRAUGHTS KING -26C4..26CD ; Extended_Pictographic# 5.2 [10] (⛄..⛍️) snowman without snow..DISABLED CAR +26A2..26B1 ; Extended_Pictographic# 4.1 [16] (⚢..⚱️) DOUBLED FEMALE SIGN..funeral urn +26B2 ; Extended_Pictographic# 5.0 [1] (⚲) NEUTER +26B3..26BC ; Extended_Pictographic# 5.1 [10] (⚳..⚼) CERES..SESQUIQUADRATE +26BD..26BF ; Extended_Pictographic# 5.2 [3] (⚽..⚿) soccer ball..SQUARED KEY +26C0..26C3 ; Extended_Pictographic# 5.1 [4] (⛀..⛃) WHITE DRAUGHTS MAN..BLACK DRAUGHTS KING +26C4..26CD ; Extended_Pictographic# 5.2 [10] (⛄..⛍) snowman without snow..DISABLED CAR 26CE ; Extended_Pictographic# 6.0 [1] (⛎) Ophiuchus -26CF..26E1 ; Extended_Pictographic# 5.2 [19] (⛏️..⛡️) pick..RESTRICTED LEFT ENTRY-2 -26E2 ; Extended_Pictographic# 6.0 [1] (⛢️) ASTRONOMICAL SYMBOL FOR URANUS -26E3 ; Extended_Pictographic# 5.2 [1] (⛣️) HEAVY CIRCLE WITH STROKE AND TWO DOTS ABOVE -26E4..26E7 ; Extended_Pictographic# 6.0 [4] (⛤️..⛧️) PENTAGRAM..INVERTED PENTAGRAM -26E8..26FF ; Extended_Pictographic# 5.2 [24] (⛨️..⛿️) BLACK CROSS ON SHIELD..WHITE FLAG WITH HORIZONTAL MIDDLE BLACK STRIPE -2700 ; Extended_Pictographic# 7.0 [1] (✀️) BLACK SAFETY SCISSORS -2701..2704 ; Extended_Pictographic# 1.1 [4] (✁️..✄️) UPPER BLADE SCISSORS..WHITE SCISSORS -2705 ; Extended_Pictographic# 6.0 [1] (✅) white heavy check mark +26CF..26E1 ; Extended_Pictographic# 5.2 [19] (⛏️..⛡) pick..RESTRICTED LEFT ENTRY-2 +26E2 ; Extended_Pictographic# 6.0 [1] (⛢) ASTRONOMICAL SYMBOL FOR URANUS +26E3 ; Extended_Pictographic# 5.2 [1] (⛣) HEAVY CIRCLE WITH STROKE AND TWO DOTS ABOVE +26E4..26E7 ; Extended_Pictographic# 6.0 [4] (⛤..⛧) PENTAGRAM..INVERTED PENTAGRAM +26E8..26FF ; Extended_Pictographic# 5.2 [24] (⛨..⛿) BLACK CROSS ON SHIELD..WHITE FLAG WITH HORIZONTAL MIDDLE BLACK STRIPE +2700 ; Extended_Pictographic# 7.0 [1] (✀) BLACK SAFETY SCISSORS +2701..2704 ; Extended_Pictographic# 1.1 [4] (✁..✄) UPPER BLADE SCISSORS..WHITE SCISSORS +2705 ; Extended_Pictographic# 6.0 [1] (✅) check mark button 2708..2709 ; Extended_Pictographic# 1.1 [2] (✈️..✉️) airplane..envelope 270A..270B ; Extended_Pictographic# 6.0 [2] (✊..✋) raised fist..raised hand 270C..2712 ; Extended_Pictographic# 1.1 [7] (✌️..✒️) victory hand..black nib -2714 ; Extended_Pictographic# 1.1 [1] (✔️) heavy check mark -2716 ; Extended_Pictographic# 1.1 [1] (✖️) heavy multiplication x +2714 ; Extended_Pictographic# 1.1 [1] (✔️) check mark +2716 ; Extended_Pictographic# 1.1 [1] (✖️) multiplication sign 271D ; Extended_Pictographic# 1.1 [1] (✝️) latin cross 2721 ; Extended_Pictographic# 1.1 [1] (✡️) star of David 2728 ; Extended_Pictographic# 6.0 [1] (✨) sparkles @@ -537,8 +571,8 @@ E0020..E007F ; Emoji_Component # 3.1 [96] (..) tag space..ca 274E ; Extended_Pictographic# 6.0 [1] (❎) cross mark button 2753..2755 ; Extended_Pictographic# 6.0 [3] (❓..❕) question mark..white exclamation mark 2757 ; Extended_Pictographic# 5.2 [1] (❗) exclamation mark -2763..2767 ; Extended_Pictographic# 1.1 [5] (❣️..❧️) heavy heart exclamation..ROTATED FLORAL HEART BULLET -2795..2797 ; Extended_Pictographic# 6.0 [3] (➕..➗) heavy plus sign..heavy division sign +2763..2767 ; Extended_Pictographic# 1.1 [5] (❣️..❧) heart exclamation..ROTATED FLORAL HEART BULLET +2795..2797 ; Extended_Pictographic# 6.0 [3] (➕..➗) plus sign..division sign 27A1 ; Extended_Pictographic# 1.1 [1] (➡️) right arrow 27B0 ; Extended_Pictographic# 6.0 [1] (➰) curly loop 27BF ; Extended_Pictographic# 6.0 [1] (➿) double curly loop @@ -546,45 +580,46 @@ E0020..E007F ; Emoji_Component # 3.1 [96] (..) tag space..ca 2B05..2B07 ; Extended_Pictographic# 4.0 [3] (⬅️..⬇️) left arrow..down arrow 2B1B..2B1C ; Extended_Pictographic# 5.1 [2] (⬛..⬜) black large square..white large square 2B50 ; Extended_Pictographic# 5.1 [1] (⭐) star -2B55 ; Extended_Pictographic# 5.2 [1] (⭕) heavy large circle +2B55 ; Extended_Pictographic# 5.2 [1] (⭕) hollow red circle 3030 ; Extended_Pictographic# 1.1 [1] (〰️) wavy dash 303D ; Extended_Pictographic# 3.2 [1] (〽️) part alternation mark 3297 ; Extended_Pictographic# 1.1 [1] (㊗️) Japanese “congratulations” button 3299 ; Extended_Pictographic# 1.1 [1] (㊙️) Japanese “secret” button -1F000..1F02B ; Extended_Pictographic# 5.1 [44] (🀀️..🀫️) MAHJONG TILE EAST WIND..MAHJONG TILE BACK -1F02C..1F02F ; Extended_Pictographic# NA [4] (️..️) <reserved-1F02C>..<reserved-1F02F> -1F030..1F093 ; Extended_Pictographic# 5.1[100] (🀰️..🂓️) DOMINO TILE HORIZONTAL BACK..DOMINO TILE VERTICAL-06-06 -1F094..1F09F ; Extended_Pictographic# NA [12] (️..️) <reserved-1F094>..<reserved-1F09F> -1F0A0..1F0AE ; Extended_Pictographic# 6.0 [15] (🂠️..🂮️) PLAYING CARD BACK..PLAYING CARD KING OF SPADES -1F0AF..1F0B0 ; Extended_Pictographic# NA [2] (️..️) <reserved-1F0AF>..<reserved-1F0B0> -1F0B1..1F0BE ; Extended_Pictographic# 6.0 [14] (🂱️..🂾️) PLAYING CARD ACE OF HEARTS..PLAYING CARD KING OF HEARTS -1F0BF ; Extended_Pictographic# 7.0 [1] (🂿️) PLAYING CARD RED JOKER -1F0C0 ; Extended_Pictographic# NA [1] (️) <reserved-1F0C0> -1F0C1..1F0CF ; Extended_Pictographic# 6.0 [15] (🃁️..🃏) PLAYING CARD ACE OF DIAMONDS..joker -1F0D0 ; Extended_Pictographic# NA [1] (️) <reserved-1F0D0> -1F0D1..1F0DF ; Extended_Pictographic# 6.0 [15] (🃑️..🃟️) PLAYING CARD ACE OF CLUBS..PLAYING CARD WHITE JOKER -1F0E0..1F0F5 ; Extended_Pictographic# 7.0 [22] (🃠️..🃵️) PLAYING CARD FOOL..PLAYING CARD TRUMP-21 -1F0F6..1F0FF ; Extended_Pictographic# NA [10] (️..️) <reserved-1F0F6>..<reserved-1F0FF> -1F10D..1F10F ; Extended_Pictographic# NA [3] (🄍️..🄏️) <reserved-1F10D>..<reserved-1F10F> -1F12F ; Extended_Pictographic# 11.0 [1] (🄯️) COPYLEFT SYMBOL -1F16C..1F16F ; Extended_Pictographic# NA [4] (🅬️..🅯️) <reserved-1F16C>..<reserved-1F16F> +1F000..1F02B ; Extended_Pictographic# 5.1 [44] (🀀..🀫) MAHJONG TILE EAST WIND..MAHJONG TILE BACK +1F02C..1F02F ; Extended_Pictographic# NA [4] (..) <reserved-1F02C>..<reserved-1F02F> +1F030..1F093 ; Extended_Pictographic# 5.1[100] (🀰..🂓) DOMINO TILE HORIZONTAL BACK..DOMINO TILE VERTICAL-06-06 +1F094..1F09F ; Extended_Pictographic# NA [12] (..) <reserved-1F094>..<reserved-1F09F> +1F0A0..1F0AE ; Extended_Pictographic# 6.0 [15] (🂠..🂮) PLAYING CARD BACK..PLAYING CARD KING OF SPADES +1F0AF..1F0B0 ; Extended_Pictographic# NA [2] (..) <reserved-1F0AF>..<reserved-1F0B0> +1F0B1..1F0BE ; Extended_Pictographic# 6.0 [14] (🂱..🂾) PLAYING CARD ACE OF HEARTS..PLAYING CARD KING OF HEARTS +1F0BF ; Extended_Pictographic# 7.0 [1] (🂿) PLAYING CARD RED JOKER +1F0C0 ; Extended_Pictographic# NA [1] () <reserved-1F0C0> +1F0C1..1F0CF ; Extended_Pictographic# 6.0 [15] (🃁..🃏) PLAYING CARD ACE OF DIAMONDS..joker +1F0D0 ; Extended_Pictographic# NA [1] () <reserved-1F0D0> +1F0D1..1F0DF ; Extended_Pictographic# 6.0 [15] (🃑..🃟) PLAYING CARD ACE OF CLUBS..PLAYING CARD WHITE JOKER +1F0E0..1F0F5 ; Extended_Pictographic# 7.0 [22] (🃠..🃵) PLAYING CARD FOOL..PLAYING CARD TRUMP-21 +1F0F6..1F0FF ; Extended_Pictographic# NA [10] (..) <reserved-1F0F6>..<reserved-1F0FF> +1F10D..1F10F ; Extended_Pictographic# NA [3] (🄍..🄏) <reserved-1F10D>..<reserved-1F10F> +1F12F ; Extended_Pictographic# 11.0 [1] (🄯) COPYLEFT SYMBOL +1F16C ; Extended_Pictographic# 12.0 [1] (🅬) RAISED MR SIGN +1F16D..1F16F ; Extended_Pictographic# NA [3] (🅭..🅯) <reserved-1F16D>..<reserved-1F16F> 1F170..1F171 ; Extended_Pictographic# 6.0 [2] (🅰️..🅱️) A button (blood type)..B button (blood type) 1F17E ; Extended_Pictographic# 6.0 [1] (🅾️) O button (blood type) 1F17F ; Extended_Pictographic# 5.2 [1] (🅿️) P button 1F18E ; Extended_Pictographic# 6.0 [1] (🆎) AB button (blood type) 1F191..1F19A ; Extended_Pictographic# 6.0 [10] (🆑..🆚) CL button..VS button -1F1AD..1F1E5 ; Extended_Pictographic# NA [57] (🆭️..️) <reserved-1F1AD>..<reserved-1F1E5> +1F1AD..1F1E5 ; Extended_Pictographic# NA [57] (🆭..) <reserved-1F1AD>..<reserved-1F1E5> 1F201..1F202 ; Extended_Pictographic# 6.0 [2] (🈁..🈂️) Japanese “here” button..Japanese “service charge” button -1F203..1F20F ; Extended_Pictographic# NA [13] (️..️) <reserved-1F203>..<reserved-1F20F> +1F203..1F20F ; Extended_Pictographic# NA [13] (..) <reserved-1F203>..<reserved-1F20F> 1F21A ; Extended_Pictographic# 5.2 [1] (🈚) Japanese “free of charge” button 1F22F ; Extended_Pictographic# 5.2 [1] (🈯) Japanese “reserved” button 1F232..1F23A ; Extended_Pictographic# 6.0 [9] (🈲..🈺) Japanese “prohibited” button..Japanese “open for business” button -1F23C..1F23F ; Extended_Pictographic# NA [4] (️..️) <reserved-1F23C>..<reserved-1F23F> -1F249..1F24F ; Extended_Pictographic# NA [7] (️..️) <reserved-1F249>..<reserved-1F24F> +1F23C..1F23F ; Extended_Pictographic# NA [4] (..) <reserved-1F23C>..<reserved-1F23F> +1F249..1F24F ; Extended_Pictographic# NA [7] (..) <reserved-1F249>..<reserved-1F24F> 1F250..1F251 ; Extended_Pictographic# 6.0 [2] (🉐..🉑) Japanese “bargain” button..Japanese “acceptable” button -1F252..1F25F ; Extended_Pictographic# NA [14] (️..️) <reserved-1F252>..<reserved-1F25F> -1F260..1F265 ; Extended_Pictographic# 10.0 [6] (🉠️..🉥️) ROUNDED SYMBOL FOR FU..ROUNDED SYMBOL FOR CAI -1F266..1F2FF ; Extended_Pictographic# NA[154] (️..️) <reserved-1F266>..<reserved-1F2FF> +1F252..1F25F ; Extended_Pictographic# NA [14] (..) <reserved-1F252>..<reserved-1F25F> +1F260..1F265 ; Extended_Pictographic# 10.0 [6] (🉠..🉥) ROUNDED SYMBOL FOR FU..ROUNDED SYMBOL FOR CAI +1F266..1F2FF ; Extended_Pictographic# NA[154] (..) <reserved-1F266>..<reserved-1F2FF> 1F300..1F320 ; Extended_Pictographic# 6.0 [33] (🌀..🌠) cyclone..shooting star 1F321..1F32C ; Extended_Pictographic# 7.0 [12] (🌡️..🌬️) thermometer..wind face 1F32D..1F32F ; Extended_Pictographic# 8.0 [3] (🌭..🌯) hot dog..burrito @@ -594,7 +629,7 @@ E0020..E007F ; Emoji_Component # 3.1 [96] (..) tag space..ca 1F37D ; Extended_Pictographic# 7.0 [1] (🍽️) fork and knife with plate 1F37E..1F37F ; Extended_Pictographic# 8.0 [2] (🍾..🍿) bottle with popping cork..popcorn 1F380..1F393 ; Extended_Pictographic# 6.0 [20] (🎀..🎓) ribbon..graduation cap -1F394..1F39F ; Extended_Pictographic# 7.0 [12] (🎔️..🎟️) HEART WITH TIP ON THE LEFT..admission tickets +1F394..1F39F ; Extended_Pictographic# 7.0 [12] (🎔..🎟️) HEART WITH TIP ON THE LEFT..admission tickets 1F3A0..1F3C4 ; Extended_Pictographic# 6.0 [37] (🎠..🏄) carousel horse..person surfing 1F3C5 ; Extended_Pictographic# 7.0 [1] (🏅) sports medal 1F3C6..1F3CA ; Extended_Pictographic# 6.0 [5] (🏆..🏊) trophy..person swimming @@ -602,7 +637,7 @@ E0020..E007F ; Emoji_Component # 3.1 [96] (..) tag space..ca 1F3CF..1F3D3 ; Extended_Pictographic# 8.0 [5] (🏏..🏓) cricket game..ping pong 1F3D4..1F3DF ; Extended_Pictographic# 7.0 [12] (🏔️..🏟️) snow-capped mountain..stadium 1F3E0..1F3F0 ; Extended_Pictographic# 6.0 [17] (🏠..🏰) house..castle -1F3F1..1F3F7 ; Extended_Pictographic# 7.0 [7] (🏱️..🏷️) WHITE PENNANT..label +1F3F1..1F3F7 ; Extended_Pictographic# 7.0 [7] (🏱..🏷️) WHITE PENNANT..label 1F3F8..1F3FA ; Extended_Pictographic# 8.0 [3] (🏸..🏺) badminton..amphora 1F400..1F43E ; Extended_Pictographic# 6.0 [63] (🐀..🐾) rat..paw prints 1F43F ; Extended_Pictographic# 7.0 [1] (🐿️) chipmunk @@ -611,15 +646,15 @@ E0020..E007F ; Emoji_Component # 3.1 [96] (..) tag space..ca 1F442..1F4F7 ; Extended_Pictographic# 6.0[182] (👂..📷) ear..camera 1F4F8 ; Extended_Pictographic# 7.0 [1] (📸) camera with flash 1F4F9..1F4FC ; Extended_Pictographic# 6.0 [4] (📹..📼) video camera..videocassette -1F4FD..1F4FE ; Extended_Pictographic# 7.0 [2] (📽️..📾️) film projector..PORTABLE STEREO +1F4FD..1F4FE ; Extended_Pictographic# 7.0 [2] (📽️..📾) film projector..PORTABLE STEREO 1F4FF ; Extended_Pictographic# 8.0 [1] (📿) prayer beads 1F500..1F53D ; Extended_Pictographic# 6.0 [62] (🔀..🔽) shuffle tracks button..downwards button -1F546..1F54A ; Extended_Pictographic# 7.0 [5] (🕆️..🕊️) WHITE LATIN CROSS..dove -1F54B..1F54F ; Extended_Pictographic# 8.0 [5] (🕋..🕏️) kaaba..BOWL OF HYGIEIA +1F546..1F54A ; Extended_Pictographic# 7.0 [5] (🕆..🕊️) WHITE LATIN CROSS..dove +1F54B..1F54F ; Extended_Pictographic# 8.0 [5] (🕋..🕏) kaaba..BOWL OF HYGIEIA 1F550..1F567 ; Extended_Pictographic# 6.0 [24] (🕐..🕧) one o’clock..twelve-thirty -1F568..1F579 ; Extended_Pictographic# 7.0 [18] (🕨️..🕹️) RIGHT SPEAKER..joystick +1F568..1F579 ; Extended_Pictographic# 7.0 [18] (🕨..🕹️) RIGHT SPEAKER..joystick 1F57A ; Extended_Pictographic# 9.0 [1] (🕺) man dancing -1F57B..1F5A3 ; Extended_Pictographic# 7.0 [41] (🕻️..🖣️) LEFT HAND TELEPHONE RECEIVER..BLACK DOWN POINTING BACKHAND INDEX +1F57B..1F5A3 ; Extended_Pictographic# 7.0 [41] (🕻..🖣) LEFT HAND TELEPHONE RECEIVER..BLACK DOWN POINTING BACKHAND INDEX 1F5A4 ; Extended_Pictographic# 9.0 [1] (🖤) black heart 1F5A5..1F5FA ; Extended_Pictographic# 7.0 [86] (🖥️..🗺️) desktop computer..world map 1F5FB..1F5FF ; Extended_Pictographic# 6.0 [5] (🗻..🗿) mount fuji..moai @@ -644,32 +679,37 @@ E0020..E007F ; Emoji_Component # 3.1 [96] (..) tag space..ca 1F62E..1F62F ; Extended_Pictographic# 6.1 [2] (😮..😯) face with open mouth..hushed face 1F630..1F633 ; Extended_Pictographic# 6.0 [4] (😰..😳) anxious face with sweat..flushed face 1F634 ; Extended_Pictographic# 6.1 [1] (😴) sleeping face -1F635..1F640 ; Extended_Pictographic# 6.0 [12] (😵..🙀) dizzy face..weary cat face +1F635..1F640 ; Extended_Pictographic# 6.0 [12] (😵..🙀) dizzy face..weary cat 1F641..1F642 ; Extended_Pictographic# 7.0 [2] (🙁..🙂) slightly frowning face..slightly smiling face 1F643..1F644 ; Extended_Pictographic# 8.0 [2] (🙃..🙄) upside-down face..face with rolling eyes 1F645..1F64F ; Extended_Pictographic# 6.0 [11] (🙅..🙏) person gesturing NO..folded hands 1F680..1F6C5 ; Extended_Pictographic# 6.0 [70] (🚀..🛅) rocket..left luggage -1F6C6..1F6CF ; Extended_Pictographic# 7.0 [10] (🛆️..🛏️) TRIANGLE WITH ROUNDED CORNERS..bed +1F6C6..1F6CF ; Extended_Pictographic# 7.0 [10] (🛆..🛏️) TRIANGLE WITH ROUNDED CORNERS..bed 1F6D0 ; Extended_Pictographic# 8.0 [1] (🛐) place of worship 1F6D1..1F6D2 ; Extended_Pictographic# 9.0 [2] (🛑..🛒) stop sign..shopping cart -1F6D3..1F6D4 ; Extended_Pictographic# 10.0 [2] (🛓️..🛔️) STUPA..PAGODA -1F6D5..1F6DF ; Extended_Pictographic# NA [11] (🛕️..🛟️) <reserved-1F6D5>..<reserved-1F6DF> +1F6D3..1F6D4 ; Extended_Pictographic# 10.0 [2] (🛓..🛔) STUPA..PAGODA +1F6D5 ; Extended_Pictographic# 12.0 [1] (🛕) hindu temple +1F6D6..1F6DF ; Extended_Pictographic# NA [10] (🛖..🛟) <reserved-1F6D6>..<reserved-1F6DF> 1F6E0..1F6EC ; Extended_Pictographic# 7.0 [13] (🛠️..🛬) hammer and wrench..airplane arrival -1F6ED..1F6EF ; Extended_Pictographic# NA [3] (️..️) <reserved-1F6ED>..<reserved-1F6EF> +1F6ED..1F6EF ; Extended_Pictographic# NA [3] (..) <reserved-1F6ED>..<reserved-1F6EF> 1F6F0..1F6F3 ; Extended_Pictographic# 7.0 [4] (🛰️..🛳️) satellite..passenger ship 1F6F4..1F6F6 ; Extended_Pictographic# 9.0 [3] (🛴..🛶) kick scooter..canoe 1F6F7..1F6F8 ; Extended_Pictographic# 10.0 [2] (🛷..🛸) sled..flying saucer 1F6F9 ; Extended_Pictographic# 11.0 [1] (🛹) skateboard -1F6FA..1F6FF ; Extended_Pictographic# NA [6] (🛺️..️) <reserved-1F6FA>..<reserved-1F6FF> -1F774..1F77F ; Extended_Pictographic# NA [12] (🝴️..🝿️) <reserved-1F774>..<reserved-1F77F> -1F7D5..1F7D8 ; Extended_Pictographic# 11.0 [4] (🟕️..🟘️) CIRCLED TRIANGLE..NEGATIVE CIRCLED SQUARE -1F7D9..1F7FF ; Extended_Pictographic# NA [39] (🟙️..️) <reserved-1F7D9>..<reserved-1F7FF> -1F80C..1F80F ; Extended_Pictographic# NA [4] (️..️) <reserved-1F80C>..<reserved-1F80F> -1F848..1F84F ; Extended_Pictographic# NA [8] (️..️) <reserved-1F848>..<reserved-1F84F> -1F85A..1F85F ; Extended_Pictographic# NA [6] (️..️) <reserved-1F85A>..<reserved-1F85F> -1F888..1F88F ; Extended_Pictographic# NA [8] (️..️) <reserved-1F888>..<reserved-1F88F> -1F8AE..1F8FF ; Extended_Pictographic# NA [82] (️..️) <reserved-1F8AE>..<reserved-1F8FF> -1F90C..1F90F ; Extended_Pictographic# NA [4] (🤌️..🤏️) <reserved-1F90C>..<reserved-1F90F> +1F6FA ; Extended_Pictographic# 12.0 [1] (🛺) auto rickshaw +1F6FB..1F6FF ; Extended_Pictographic# NA [5] (🛻..) <reserved-1F6FB>..<reserved-1F6FF> +1F774..1F77F ; Extended_Pictographic# NA [12] (🝴..🝿) <reserved-1F774>..<reserved-1F77F> +1F7D5..1F7D8 ; Extended_Pictographic# 11.0 [4] (🟕..🟘) CIRCLED TRIANGLE..NEGATIVE CIRCLED SQUARE +1F7D9..1F7DF ; Extended_Pictographic# NA [7] (🟙..) <reserved-1F7D9>..<reserved-1F7DF> +1F7E0..1F7EB ; Extended_Pictographic# 12.0 [12] (🟠..🟫) orange circle..brown square +1F7EC..1F7FF ; Extended_Pictographic# NA [20] (..) <reserved-1F7EC>..<reserved-1F7FF> +1F80C..1F80F ; Extended_Pictographic# NA [4] (..) <reserved-1F80C>..<reserved-1F80F> +1F848..1F84F ; Extended_Pictographic# NA [8] (..) <reserved-1F848>..<reserved-1F84F> +1F85A..1F85F ; Extended_Pictographic# NA [6] (..) <reserved-1F85A>..<reserved-1F85F> +1F888..1F88F ; Extended_Pictographic# NA [8] (..) <reserved-1F888>..<reserved-1F88F> +1F8AE..1F8FF ; Extended_Pictographic# NA [82] (..) <reserved-1F8AE>..<reserved-1F8FF> +1F90C ; Extended_Pictographic# NA [1] (🤌) <reserved-1F90C> +1F90D..1F90F ; Extended_Pictographic# 12.0 [3] (🤍..🤏) white heart..pinching hand 1F910..1F918 ; Extended_Pictographic# 8.0 [9] (🤐..🤘) zipper-mouth face..sign of the horns 1F919..1F91E ; Extended_Pictographic# 9.0 [6] (🤙..🤞) call me hand..crossed fingers 1F91F ; Extended_Pictographic# 10.0 [1] (🤟) love-you gesture @@ -679,35 +719,50 @@ E0020..E007F ; Emoji_Component # 3.1 [96] (..) tag space..ca 1F931..1F932 ; Extended_Pictographic# 10.0 [2] (🤱..🤲) breast-feeding..palms up together 1F933..1F93A ; Extended_Pictographic# 9.0 [8] (🤳..🤺) selfie..person fencing 1F93C..1F93E ; Extended_Pictographic# 9.0 [3] (🤼..🤾) people wrestling..person playing handball -1F93F ; Extended_Pictographic# NA [1] (🤿️) <reserved-1F93F> +1F93F ; Extended_Pictographic# 12.0 [1] (🤿) diving mask 1F940..1F945 ; Extended_Pictographic# 9.0 [6] (🥀..🥅) wilted flower..goal net 1F947..1F94B ; Extended_Pictographic# 9.0 [5] (🥇..🥋) 1st place medal..martial arts uniform 1F94C ; Extended_Pictographic# 10.0 [1] (🥌) curling stone 1F94D..1F94F ; Extended_Pictographic# 11.0 [3] (🥍..🥏) lacrosse..flying disc 1F950..1F95E ; Extended_Pictographic# 9.0 [15] (🥐..🥞) croissant..pancakes 1F95F..1F96B ; Extended_Pictographic# 10.0 [13] (🥟..🥫) dumpling..canned food -1F96C..1F970 ; Extended_Pictographic# 11.0 [5] (🥬..🥰) leafy green..smiling face with 3 hearts -1F971..1F972 ; Extended_Pictographic# NA [2] (🥱️..🥲️) <reserved-1F971>..<reserved-1F972> +1F96C..1F970 ; Extended_Pictographic# 11.0 [5] (🥬..🥰) leafy green..smiling face with hearts +1F971 ; Extended_Pictographic# 12.0 [1] (🥱) yawning face +1F972 ; Extended_Pictographic# NA [1] (🥲) <reserved-1F972> 1F973..1F976 ; Extended_Pictographic# 11.0 [4] (🥳..🥶) partying face..cold face -1F977..1F979 ; Extended_Pictographic# NA [3] (🥷️..🥹️) <reserved-1F977>..<reserved-1F979> +1F977..1F979 ; Extended_Pictographic# NA [3] (🥷..🥹) <reserved-1F977>..<reserved-1F979> 1F97A ; Extended_Pictographic# 11.0 [1] (🥺) pleading face -1F97B ; Extended_Pictographic# NA [1] (🥻️) <reserved-1F97B> -1F97C..1F97F ; Extended_Pictographic# 11.0 [4] (🥼..🥿) lab coat..woman’s flat shoe -1F980..1F984 ; Extended_Pictographic# 8.0 [5] (🦀..🦄) crab..unicorn face +1F97B ; Extended_Pictographic# 12.0 [1] (🥻) sari +1F97C..1F97F ; Extended_Pictographic# 11.0 [4] (🥼..🥿) lab coat..flat shoe +1F980..1F984 ; Extended_Pictographic# 8.0 [5] (🦀..🦄) crab..unicorn 1F985..1F991 ; Extended_Pictographic# 9.0 [13] (🦅..🦑) eagle..squid 1F992..1F997 ; Extended_Pictographic# 10.0 [6] (🦒..🦗) giraffe..cricket 1F998..1F9A2 ; Extended_Pictographic# 11.0 [11] (🦘..🦢) kangaroo..swan -1F9A3..1F9AF ; Extended_Pictographic# NA [13] (🦣️..🦯️) <reserved-1F9A3>..<reserved-1F9AF> -1F9B0..1F9B9 ; Extended_Pictographic# 11.0 [10] (🦰..🦹) red-haired..supervillain -1F9BA..1F9BF ; Extended_Pictographic# NA [6] (🦺️..🦿️) <reserved-1F9BA>..<reserved-1F9BF> +1F9A3..1F9A4 ; Extended_Pictographic# NA [2] (🦣..🦤) <reserved-1F9A3>..<reserved-1F9A4> +1F9A5..1F9AA ; Extended_Pictographic# 12.0 [6] (🦥..🦪) sloth..oyster +1F9AB..1F9AD ; Extended_Pictographic# NA [3] (🦫..🦭) <reserved-1F9AB>..<reserved-1F9AD> +1F9AE..1F9AF ; Extended_Pictographic# 12.0 [2] (🦮..🦯) guide dog..probing cane +1F9B0..1F9B9 ; Extended_Pictographic# 11.0 [10] (🦰..🦹) red hair..supervillain +1F9BA..1F9BF ; Extended_Pictographic# 12.0 [6] (🦺..🦿) safety vest..mechanical leg 1F9C0 ; Extended_Pictographic# 8.0 [1] (🧀) cheese wedge 1F9C1..1F9C2 ; Extended_Pictographic# 11.0 [2] (🧁..🧂) cupcake..salt -1F9C3..1F9CF ; Extended_Pictographic# NA [13] (🧃️..🧏️) <reserved-1F9C3>..<reserved-1F9CF> +1F9C3..1F9CA ; Extended_Pictographic# 12.0 [8] (🧃..🧊) beverage box..ice cube +1F9CB..1F9CC ; Extended_Pictographic# NA [2] (🧋..🧌) <reserved-1F9CB>..<reserved-1F9CC> +1F9CD..1F9CF ; Extended_Pictographic# 12.0 [3] (🧍..🧏) person standing..deaf person 1F9D0..1F9E6 ; Extended_Pictographic# 10.0 [23] (🧐..🧦) face with monocle..socks 1F9E7..1F9FF ; Extended_Pictographic# 11.0 [25] (🧧..🧿) red envelope..nazar amulet -1FA00..1FA5F ; Extended_Pictographic# NA [96] (🨀️..️) <reserved-1FA00>..<reserved-1FA5F> -1FA60..1FA6D ; Extended_Pictographic# 11.0 [14] (🩠️..🩭️) XIANGQI RED GENERAL..XIANGQI BLACK SOLDIER -1FA6E..1FFFD ; Extended_Pictographic# NA[1424] (️..️) <reserved-1FA6E>..<reserved-1FFFD> +1FA00..1FA53 ; Extended_Pictographic# 12.0 [84] (🨀..🩓) NEUTRAL CHESS KING..BLACK CHESS KNIGHT-BISHOP +1FA54..1FA5F ; Extended_Pictographic# NA [12] (..) <reserved-1FA54>..<reserved-1FA5F> +1FA60..1FA6D ; Extended_Pictographic# 11.0 [14] (🩠..🩭) XIANGQI RED GENERAL..XIANGQI BLACK SOLDIER +1FA6E..1FA6F ; Extended_Pictographic# NA [2] (..) <reserved-1FA6E>..<reserved-1FA6F> +1FA70..1FA73 ; Extended_Pictographic# 12.0 [4] (🩰..🩳) ballet shoes..shorts +1FA74..1FA77 ; Extended_Pictographic# NA [4] (🩴..🩷) <reserved-1FA74>..<reserved-1FA77> +1FA78..1FA7A ; Extended_Pictographic# 12.0 [3] (🩸..🩺) drop of blood..stethoscope +1FA7B..1FA7F ; Extended_Pictographic# NA [5] (🩻..) <reserved-1FA7B>..<reserved-1FA7F> +1FA80..1FA82 ; Extended_Pictographic# 12.0 [3] (🪀..🪂) yo-yo..parachute +1FA83..1FA8F ; Extended_Pictographic# NA [13] (🪃..) <reserved-1FA83>..<reserved-1FA8F> +1FA90..1FA95 ; Extended_Pictographic# 12.0 [6] (🪐..🪕) ringed planet..banjo +1FA96..1FFFD ; Extended_Pictographic# NA[1384] (🪖..) <reserved-1FA96>..<reserved-1FFFD> # Total elements: 3793 diff --git a/lib/stdlib/uc_spec/gen_unicode_mod.escript b/lib/stdlib/uc_spec/gen_unicode_mod.escript index de67b18afc..d820b9ed8e 100644 --- a/lib/stdlib/uc_spec/gen_unicode_mod.escript +++ b/lib/stdlib/uc_spec/gen_unicode_mod.escript @@ -191,7 +191,7 @@ gen_static(Fd) -> " {U,L} -> #{upper=>U,lower=>L,title=>U,fold=>L};\n" " {U,L,T,F} -> #{upper=>U,lower=>L,title=>T,fold=>F}\n" " end.\n\n"), - io:put_chars(Fd, "spec_version() -> {11,0}.\n\n\n"), + io:put_chars(Fd, "spec_version() -> {12,1}.\n\n\n"), io:put_chars(Fd, "class(Codepoint) -> {CCC,_,_} = unicode_table(Codepoint),\n CCC.\n\n"), io:put_chars(Fd, "-spec uppercase(unicode:chardata()) -> " "maybe_improper_list(gc(),unicode:chardata()).\n"), diff --git a/lib/syntax_tools/Makefile b/lib/syntax_tools/Makefile index 14ae6d4f97..d3e2aa9b2c 100644 --- a/lib/syntax_tools/Makefile +++ b/lib/syntax_tools/Makefile @@ -91,3 +91,5 @@ tar: $(APP_TAR_FILE) $(APP_TAR_FILE): $(APP_DIR) (cd $(APP_RELEASE_DIR); gtar zcf $(APP_TAR_FILE) $(DIR_NAME)) + +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/syntax_tools/src/epp_dodger.erl b/lib/syntax_tools/src/epp_dodger.erl index 7e741cc649..da22a91de0 100644 --- a/lib/syntax_tools/src/epp_dodger.erl +++ b/lib/syntax_tools/src/epp_dodger.erl @@ -598,8 +598,6 @@ skip_macro_args([{'receive',_}=T | Ts], Es, As) -> skip_macro_args(Ts, ['end' | Es], [T | As]); skip_macro_args([{'try',_}=T | Ts], Es, As) -> skip_macro_args(Ts, ['end' | Es], [T | As]); -skip_macro_args([{'cond',_}=T | Ts], Es, As) -> - skip_macro_args(Ts, ['end' | Es], [T | As]); skip_macro_args([{E,_}=T | Ts], [E], As) -> %final close {lists:reverse([T | As]), Ts}; skip_macro_args([{E,_}=T | Ts], [E | Es], As) -> %matching close diff --git a/lib/syntax_tools/src/erl_prettypr.erl b/lib/syntax_tools/src/erl_prettypr.erl index 6ad9bec2e6..d70dd40a8a 100644 --- a/lib/syntax_tools/src/erl_prettypr.erl +++ b/lib/syntax_tools/src/erl_prettypr.erl @@ -53,7 +53,7 @@ -type hook() :: 'none' | fun((erl_syntax:syntaxTree(), _, _) -> prettypr:document()). --type clause_t() :: 'case_expr' | 'cond_expr' | 'fun_expr' +-type clause_t() :: 'case_expr' | 'fun_expr' | 'if_expr' | 'receive_expr' | 'try_expr' | {'function', prettypr:document()} | 'spec'. @@ -586,8 +586,6 @@ lay_2(Node, Ctxt) -> make_fun_clause(N, D1, D2, D3, Ctxt); if_expr -> make_if_clause(D1, D2, D3, Ctxt); - cond_expr -> - make_if_clause(D1, D2, D3, Ctxt); case_expr -> make_case_clause(D1, D2, D3, Ctxt); receive_expr -> @@ -627,14 +625,6 @@ lay_2(Node, Ctxt) -> sep([follow(text("if"), D, Ctxt1#ctxt.sub_indent), text("end")]); - cond_expr -> - Ctxt1 = reset_prec(Ctxt), - D = lay_clauses(erl_syntax:cond_expr_clauses(Node), - cond_expr, Ctxt1), - sep([text("cond"), - nest(Ctxt1#ctxt.sub_indent, D), - text("end")]); - fun_expr -> Ctxt1 = reset_prec(Ctxt), D = lay_clauses(erl_syntax:fun_expr_clauses(Node), diff --git a/lib/syntax_tools/src/erl_syntax.erl b/lib/syntax_tools/src/erl_syntax.erl index 09ef0bf7a5..ed94bd383c 100644 --- a/lib/syntax_tools/src/erl_syntax.erl +++ b/lib/syntax_tools/src/erl_syntax.erl @@ -183,8 +183,6 @@ comment/2, comment_padding/1, comment_text/1, - cond_expr/1, - cond_expr_clauses/1, conjunction/1, conjunction_body/1, constrained_function_type/2, @@ -431,6 +429,7 @@ -record(tree, {type :: atom(), attr = #attr{} :: #attr{}, data :: term()}). +-type tree() :: #tree{}. %% `wrapper' records are used for attaching new-form node information to %% `erl_parse' trees. @@ -446,18 +445,20 @@ -record(wrapper, {type :: atom(), attr = #attr{} :: #attr{}, tree :: erl_parse()}). +-type wrapper() :: #wrapper{}. %% ===================================================================== --type syntaxTree() :: #tree{} | #wrapper{} | erl_parse(). +-type syntaxTree() :: tree() | wrapper() | erl_parse(). -type erl_parse() :: erl_parse:abstract_clause() | erl_parse:abstract_expr() | erl_parse:abstract_form() | erl_parse:abstract_type() | erl_parse:form_info() - %% To shut up Dialyzer: - | {bin_element, _, _, _, _}. + | erl_parse:af_binelement(term()) + | erl_parse:af_generator() + | erl_parse:af_remote_function(). %% The representation built by the Erlang standard library parser %% `erl_parse'. This is a subset of the {@link syntaxTree()} type. @@ -494,39 +495,38 @@ %% <td>class_qualifier</td> %% <td>clause</td> %% <td>comment</td> -%% <td>cond_expr</td> -%% </tr><tr> %% <td>conjunction</td> +%% </tr><tr> %% <td>constrained_function_type</td> %% <td>constraint</td> %% <td>disjunction</td> -%% </tr><tr> %% <td>eof_marker</td> +%% </tr><tr> %% <td>error_marker</td> %% <td>float</td> %% <td>form_list</td> -%% </tr><tr> %% <td>fun_expr</td> +%% </tr><tr> %% <td>fun_type</td> %% <td>function</td> %% <td>function_type</td> -%% </tr><tr> %% <td>generator</td> +%% </tr><tr> %% <td>if_expr</td> %% <td>implicit_fun</td> %% <td>infix_expr</td> -%% </tr><tr> %% <td>integer</td> +%% </tr><tr> %% <td>integer_range_type</td> %% <td>list</td> %% <td>list_comp</td> -%% </tr><tr> %% <td>macro</td> +%% </tr><tr> %% <td>map_expr</td> %% <td>map_field_assoc</td> %% <td>map_field_exact</td> -%% </tr><tr> %% <td>map_type</td> +%% </tr><tr> %% <td>map_type_assoc</td> %% <td>map_type_exact</td> %% <td>match_expr</td> @@ -556,6 +556,7 @@ %% <td>tuple_type</td> %% <td>typed_record_field</td> %% <td>type_application</td> +%% </tr><tr> %% <td>type_union</td> %% <td>underscore</td> %% <td>user_type_application</td> @@ -587,7 +588,6 @@ %% @see class_qualifier/2 %% @see clause/3 %% @see comment/2 -%% @see cond_expr/1 %% @see conjunction/1 %% @see constrained_function_type/2 %% @see constraint/2 @@ -673,7 +673,6 @@ type(Node) -> %% Composite types {'case', _, _, _} -> case_expr; {'catch', _, _} -> catch_expr; - {'cond', _, _} -> cond_expr; {'fun', _, {clauses, _}} -> fun_expr; {named_fun, _, _, _} -> named_fun_expr; {'fun', _, {function, _, _}} -> implicit_fun; @@ -6290,7 +6289,6 @@ if_expr_clauses(Node) -> %% @see case_expr_argument/1 %% @see clause/3 %% @see if_expr/1 -%% @see cond_expr/1 -record(case_expr, {argument :: syntaxTree(), clauses :: [syntaxTree()]}). @@ -6357,60 +6355,6 @@ case_expr_clauses(Node) -> %% ===================================================================== -%% @doc Creates an abstract cond-expression. If `Clauses' is -%% `[C1, ..., Cn]', the result represents "<code>cond -%% <em>C1</em>; ...; <em>Cn</em> end</code>". More exactly, if each -%% `Ci' represents "<code>() <em>Ei</em> -> -%% <em>Bi</em></code>", then the result represents "<code>cond -%% <em>E1</em> -> <em>B1</em>; ...; <em>En</em> -> <em>Bn</em> -%% end</code>". -%% -%% @see cond_expr_clauses/1 -%% @see clause/3 -%% @see case_expr/2 - -%% type(Node) = cond_expr -%% data(Node) = Clauses -%% -%% Clauses = [syntaxTree()] -%% -%% `erl_parse' representation: -%% -%% {'cond', Pos, Clauses} -%% -%% Clauses = [Clause] \ [] -%% Clause = {clause, ...} -%% -%% See `clause' for documentation on `erl_parse' clauses. - --spec cond_expr([syntaxTree()]) -> syntaxTree(). - -cond_expr(Clauses) -> - tree(cond_expr, Clauses). - -revert_cond_expr(Node) -> - Pos = get_pos(Node), - Clauses = [revert_clause(C) || C <- cond_expr_clauses(Node)], - {'cond', Pos, Clauses}. - - -%% ===================================================================== -%% @doc Returns the list of clause subtrees of a `cond_expr' node. -%% -%% @see cond_expr/1 - --spec cond_expr_clauses(syntaxTree()) -> [syntaxTree()]. - -cond_expr_clauses(Node) -> - case unwrap(Node) of - {'cond', _, Clauses} -> - Clauses; - Node1 -> - data(Node1) - end. - - -%% ===================================================================== %% @equiv receive_expr(Clauses, none, []) -spec receive_expr([syntaxTree()]) -> syntaxTree(). @@ -7534,8 +7478,6 @@ revert_root(Node) -> revert_char(Node); clause -> revert_clause(Node); - cond_expr -> - revert_cond_expr(Node); constrained_function_type -> revert_constrained_function_type(Node); constraint -> @@ -7802,8 +7744,6 @@ subtrees(T) -> [clause_patterns(T), [G], clause_body(T)] end; - cond_expr -> - [cond_expr_clauses(T)]; conjunction -> [conjunction_body(T)]; constrained_function_type -> @@ -8017,7 +7957,6 @@ make_tree(class_qualifier, [[A], [B]]) -> class_qualifier(A, B); make_tree(class_qualifier, [[A], [B], [C]]) -> class_qualifier(A, B, C); make_tree(clause, [P, B]) -> clause(P, none, B); make_tree(clause, [P, [G], B]) -> clause(P, G, B); -make_tree(cond_expr, [C]) -> cond_expr(C); make_tree(conjunction, [E]) -> conjunction(E); make_tree(constrained_function_type, [[F],C]) -> constrained_function_type(F, C); @@ -8239,7 +8178,7 @@ meta_call(F, As) -> %% ===================================================================== %% @equiv tree(Type, []) --spec tree(atom()) -> #tree{}. +-spec tree(atom()) -> tree(). tree(Type) -> tree(Type, []). @@ -8274,7 +8213,7 @@ tree(Type) -> %% @see data/1 %% @see type/1 --spec tree(atom(), term()) -> #tree{}. +-spec tree(atom(), term()) -> tree(). tree(Type, Data) -> #tree{type = Type, data = Data}. @@ -8330,7 +8269,7 @@ data(T) -> erlang:error({badarg, T}). %% trees. <em>Attaching a wrapper onto another wrapper structure is an %% error</em>. --spec wrap(erl_parse()) -> #wrapper{}. +-spec wrap(erl_parse()) -> wrapper(). wrap(Node) -> %% We assume that Node is an old-school `erl_parse' tree. @@ -8344,7 +8283,7 @@ wrap(Node) -> %% `erl_parse' tree; otherwise it returns `Node' %% itself. --spec unwrap(syntaxTree()) -> #tree{} | erl_parse(). +-spec unwrap(syntaxTree()) -> tree() | erl_parse(). unwrap(#wrapper{tree = Node}) -> Node; unwrap(Node) -> Node. % This could also be a new-form node. diff --git a/lib/syntax_tools/src/erl_syntax_lib.erl b/lib/syntax_tools/src/erl_syntax_lib.erl index 352165893f..6185007235 100644 --- a/lib/syntax_tools/src/erl_syntax_lib.erl +++ b/lib/syntax_tools/src/erl_syntax_lib.erl @@ -528,8 +528,6 @@ vann(Tree, Env) -> vann_case_expr(Tree, Env); if_expr -> vann_if_expr(Tree, Env); - cond_expr -> - vann_cond_expr(Tree, Env); receive_expr -> vann_receive_expr(Tree, Env); catch_expr -> @@ -613,9 +611,6 @@ vann_if_expr(Tree, Env) -> Tree1 = rewrite(Tree, erl_syntax:if_expr(Cs1)), {ann_bindings(Tree1, Env, Bound, Free), Bound, Free}. -vann_cond_expr(_Tree, _Env) -> - erlang:error({not_implemented,cond_expr}). - vann_catch_expr(Tree, Env) -> E = erl_syntax:catch_expr_body(Tree), {E1, _, Free} = vann(E, Env), diff --git a/lib/syntax_tools/src/erl_tidy.erl b/lib/syntax_tools/src/erl_tidy.erl index 5623aa6af3..1ced48ecb3 100644 --- a/lib/syntax_tools/src/erl_tidy.erl +++ b/lib/syntax_tools/src/erl_tidy.erl @@ -1551,18 +1551,6 @@ visit_match_body(Ps, P, B, Tree, Env, St0) -> false -> visit_match_expr_final(P, B, Tree, Env, St0) end; - cond_expr -> - Cs = erl_syntax:cond_expr_clauses(B), - case multival_clauses(Cs, length(Ps), Ps) of - {true, Cs1} -> - report_export_vars(Env#env.file, - erl_syntax:get_pos(B), - "cond", Env#env.verbosity), - Tree1 = erl_syntax:cond_expr(Cs1), - {rewrite(Tree, Tree1), St0}; - false -> - visit_match_expr_final(P, B, Tree, Env, St0) - end; receive_expr -> %% Handle the timeout case as an extra clause. As = erl_syntax:receive_expr_action(B), diff --git a/lib/tftp/Makefile b/lib/tftp/Makefile index a4559fbc2e..d0397beaa0 100644 --- a/lib/tftp/Makefile +++ b/lib/tftp/Makefile @@ -43,36 +43,6 @@ include $(ERL_TOP)/make/otp_subdir.mk .PHONY: info gclean dialyzer dialyzer_plt dclean -info: - @echo "OS: $(OS)" - @echo "DOCB: $(DOCB)" - @echo "" - @echo "TFTP_VSN: $(TFTP_VSN)" - @echo "APP_VSN: $(APP_VSN)" - @echo "" - @echo "DIA_PLT: $(DIA_PLT)" - @echo "DIA_ANALYSIS: $(DIA_ANALYSIS)" - @echo "" +DIA_PLT_APPS= -gclean: - git clean -fXd - -dclean: - rm -f $(DIA_PLT) - rm -f $(DIA_ANALYSIS) - -dialyzer_plt: $(DIA_PLT) - -$(DIA_PLT): - @echo "Building $(APPLICATION) plt file" - @dialyzer --build_plt \ - --output_plt $@ \ - -r ../$(APPLICATION)/ebin \ - --output $(DIA_ANALYSIS) \ - --verbose - -dialyzer: $(DIA_PLT) - @echo "Running dialyzer on $(APPLICATION)" - @dialyzer --plt $< \ - ../$(APPLICATION)/ebin \ - --verbose +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/tools/Makefile b/lib/tools/Makefile index e17e9cfd1e..d849989a2d 100644 --- a/lib/tools/Makefile +++ b/lib/tools/Makefile @@ -36,3 +36,6 @@ SPECIAL_TARGETS = # ---------------------------------------------------- include $(ERL_TOP)/make/otp_subdir.mk +DIA_PLT_APPS=compiler runtime_tools + +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/tools/test/cprof_SUITE.erl b/lib/tools/test/cprof_SUITE.erl index 9cbc27fb17..39239a66a9 100644 --- a/lib/tools/test/cprof_SUITE.erl +++ b/lib/tools/test/cprof_SUITE.erl @@ -211,16 +211,12 @@ on_load_test(Config) -> Lr = seq_r(1, M, fun succ/1), N2 = cprof:pause(), {Module,0,[]} = cprof:analyse(Module), - M_1 = M - 1, M4__4 = M*4 - 4, M10_7 = M*10 - 7, {?MODULE,M10_7,[{{?MODULE,succ,1},M4__4}, + {{?MODULE,'-fun.succ/1-',1},M4__4}, {{?MODULE,seq_r,4},M}, {{?MODULE,seq,3},M}, - {{?MODULE,'-on_load_test/1-fun-5-',1},M_1}, - {{?MODULE,'-on_load_test/1-fun-4-',1},M_1}, - {{?MODULE,'-on_load_test/1-fun-3-',1},M_1}, - {{?MODULE,'-on_load_test/1-fun-2-',1},M_1}, {{?MODULE,seq_r,3},1}]} = cprof:analyse(?MODULE), N2 = cprof:stop(), @@ -246,18 +242,14 @@ modules_test(Config) -> Lr = seq_r(1, M, fun succ/1), N = cprof:pause(), Lr = lists:reverse(L), - M_1 = M - 1, M4_4 = M*4 - 4, M10_7 = M*10 - 7, M2__1 = M*2 + 1, {Tot,ModList} = cprof:analyse(), {value,{?MODULE,M10_7,[{{?MODULE,succ,1},M4_4}, + {{?MODULE,'-fun.succ/1-',1},M4_4}, {{?MODULE,seq_r,4},M}, {{?MODULE,seq,3},M}, - {{?MODULE,'-modules_test/1-fun-3-',1},M_1}, - {{?MODULE,'-modules_test/1-fun-2-',1},M_1}, - {{?MODULE,'-modules_test/1-fun-1-',1},M_1}, - {{?MODULE,'-modules_test/1-fun-0-',1},M_1}, {{?MODULE,seq_r,3},1}]}} = lists:keysearch(?MODULE, 1, ModList), {value,{Module,M2__1,[{{Module,seq_r,4},M}, diff --git a/lib/wx/Makefile b/lib/wx/Makefile index 2397950925..002887d9da 100644 --- a/lib/wx/Makefile +++ b/lib/wx/Makefile @@ -40,3 +40,5 @@ CLEANDIRS = $(SUBDIRS) api_gen SUB_DIRECTORIES=$(SUBDIRS) include $(ERL_TOP)/make/otp_subdir.mk + +include $(ERL_TOP)/make/app_targets.mk diff --git a/lib/xmerl/Makefile b/lib/xmerl/Makefile index a584aacbac..65a0af9e4a 100644 --- a/lib/xmerl/Makefile +++ b/lib/xmerl/Makefile @@ -50,12 +50,7 @@ SPECIAL_TARGETS = include $(ERL_TOP)/make/otp_subdir.mk -.PHONY: info version - -info: - @echo "APP_RELEASE_DIR: $(APP_RELEASE_DIR)" - @echo "APP_DIR: $(APP_DIR)" - @echo "APP_TAR_FILE: $(APP_TAR_FILE)" +.PHONY: version version: @echo "$(VSN)" @@ -97,3 +92,4 @@ tar: $(APP_TAR_FILE) $(APP_TAR_FILE): $(APP_DIR) (cd $(APP_RELEASE_DIR); gtar zcf $(APP_TAR_FILE) $(DIR_NAME)) +include $(ERL_TOP)/make/app_targets.mk diff --git a/make/app_targets.mk b/make/app_targets.mk new file mode 100644 index 0000000000..e9aaa4193c --- /dev/null +++ b/make/app_targets.mk @@ -0,0 +1,66 @@ +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 1997-2019. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# %CopyrightEnd% +# + +APPLICATION ?= $(basename $(notdir $(PWD))) + +.PHONY: test info gclean dialyzer dialyzer_plt dclean + +test: + $(ERL_TOP)/make/test_target_script.sh $(ERL_TOP) + +info: + @echo "$(APPLICATION)_VSN: $(VSN)" + @echo "APP_VSN: $(APP_VSN)" + @echo "" + @echo "DIA_PLT: $(DIA_PLT)" + @echo "DIA_ANALYSIS: $(DIA_ANALYSIS)" + @echo "" + +gclean: + git clean -fXd + + +DIA_DEFAULT_PLT_APPS = erts kernel stdlib $(APPLICATION) +DIA_PLT_DIR = ./priv/plt +DIA_PLT = $(DIA_PLT_DIR)/$(APPLICATION).plt +DIA_ANALYSIS = $(basename $(DIA_PLT)).dialyzer_analysis + +dialyzer_plt: $(DIA_PLT) + +$(DIA_PLT_DIR): + @mkdir -p $@ + +$(DIA_PLT): $(DIA_PLT_DIR) + @echo "Building $(APPLICATION) plt file" + @$(ERL_TOP)/bin/dialyzer --build_plt \ + --output_plt $@ \ + --apps $(sort $(DIA_PLT_APPS) $(DIA_DEFAULT_PLT_APPS)) \ + --output $(DIA_ANALYSIS) \ + --verbose + +dialyzer: $(DIA_PLT) + @echo "Running dialyzer on $(APPLICATION)" + @dialyzer --plt $< \ + ../$(APPLICATION)/ebin \ + --verbose + +dclean: + rm -f $(DIA_PLT) + rm -f $(DIA_ANALYSIS) diff --git a/make/otp_version_tickets_in_merge b/make/otp_version_tickets_in_merge index e69de29bb2..a8bd952e0e 100644 --- a/make/otp_version_tickets_in_merge +++ b/make/otp_version_tickets_in_merge @@ -0,0 +1,125 @@ +OTP-10400 +OTP-13872 +OTP-15328 +OTP-15330 +OTP-15331 +OTP-15332 +OTP-15344 +OTP-15370 +OTP-15395 +OTP-15422 +OTP-15431 +OTP-15510 +OTP-15566 +OTP-15731 +OTP-15738 +OTP-15747 +OTP-15764 +OTP-15765 +OTP-15778 +OTP-15789 +OTP-15807 +OTP-15813 +OTP-15814 +OTP-15817 +OTP-15818 +OTP-15820 +OTP-15822 +OTP-15823 +OTP-15826 +OTP-15827 +OTP-15830 +OTP-15831 +OTP-15833 +OTP-15836 +OTP-15843 +OTP-15849 +OTP-15850 +OTP-15851 +OTP-15852 +OTP-15853 +OTP-15854 +OTP-15858 +OTP-15859 +OTP-15863 +OTP-15869 +OTP-15870 +OTP-15874 +OTP-15876 +OTP-15880 +OTP-15881 +OTP-15882 +OTP-15883 +OTP-15884 +OTP-15889 +OTP-15893 +OTP-15897 +OTP-15901 +OTP-15904 +OTP-15905 +OTP-15906 +OTP-15911 +OTP-15916 +OTP-15917 +OTP-15918 +OTP-15923 +OTP-15924 +OTP-15926 +OTP-15927 +OTP-15928 +OTP-15929 +OTP-15931 +OTP-15932 +OTP-15933 +OTP-15934 +OTP-15935 +OTP-15942 +OTP-15954 +OTP-15955 +OTP-15958 +OTP-15959 +OTP-15962 +OTP-15963 +OTP-15966 +OTP-15968 +OTP-15969 +OTP-15970 +OTP-15971 +OTP-15974 +OTP-15975 +OTP-15977 +OTP-15978 +OTP-15979 +OTP-15980 +OTP-15982 +OTP-15983 +OTP-15984 +OTP-15985 +OTP-15987 +OTP-15992 +OTP-15996 +OTP-15997 +OTP-16000 +OTP-16001 +OTP-16002 +OTP-16006 +OTP-16012 +OTP-16022 +OTP-16023 +OTP-16027 +OTP-16028 +OTP-16032 +OTP-16035 +OTP-16036 +OTP-16037 +OTP-16038 +OTP-16040 +OTP-16041 +OTP-16042 +OTP-16044 +OTP-16049 +OTP-16050 +OTP-16051 +OTP-16056 +OTP-16058 +OTP-16060 diff --git a/make/test_target_script.sh b/make/test_target_script.sh new file mode 100755 index 0000000000..f605efa120 --- /dev/null +++ b/make/test_target_script.sh @@ -0,0 +1,254 @@ +#!/bin/sh + +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 1997-2019. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# %CopyrightEnd% +# + + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +LIGHT_CYAN='\033[1;36m' +BOLD='\033[1m' +NC='\033[0m' + + +print_highlighted_msg_with_printer () { + COLOR=$1 + MSG_PRINTER=$2 + printf "\n${COLOR}======================================================================${NC}\n" + echo + $MSG_PRINTER + echo + printf "${COLOR}======================================================================${NC}\n" +} + +print_highlighted_msg () { + COLOR=$1 + MSG=$2 + print_msg () { + echo "$MSG" + } + print_highlighted_msg_with_printer $COLOR print_msg +} + +print_all_tests_takes_long_time_warning () { + print_msg () { + cat << EOM + +WARNING + +All tests will require several hours to run. You may want to check the +following text file that describes how to run tests for a specific +application. + +EOM + echo $ERL_TOP/HOWTO/TESTING.md + } + print_highlighted_msg_with_printer $YELLOW print_msg +} + +print_all_tests_for_application_notes () { + print_msg () { + cat << EOM + +NOTE 1 + +ct_run will now attempt to execute tests in the test directory, which +may take a long time to do. One can pass arguments to ct_run by +setting the ARGS variable when invoking "make test". + +Example: + +make ARGS="-suite asn1_SUITE -case ticket_7407" test + +NOTE 2 + +You may want to look at the more established way of running tests that +is described in the following text file if you encounter strange +errors: + +EOM + echo "$ERL_TOP/HOWTO/TESTING.md" + } + print_highlighted_msg_with_printer $LIGHT_CYAN print_msg +} + +print_c_files_warning () { + print_msg () { + cat << EOM + +WARNING + +The test directory contains .c files which means that some test cases +will probably not work correctly when run through "make test". The +text file at the following location describes how one can compile and +run all test cases: + + +EOM + echo $ERL_TOP/HOWTO/TESTING.md + } + print_highlighted_msg_with_printer $YELLOW print_msg +} + + +print_on_error_note () { + print_msg () { + cat << EOM +NOTE: + +Some test cases do not work correctly when run through "make test" as +they are designed to be run through the method that is described in +the "$ERL_TOP/HOWTO/TESTING.md" text file. You may want to check this +text file if you encounter strange errors. Note also that you can +rerun a specific test case by passing parameters to ct_run as in the +example below: + +make ARGS="-suite asn1_SUITE -case ticket_7407" test + +EOM + } + print_highlighted_msg_with_printer $NC print_msg +} + +# Check ERL_TOP + +if [ -d "$1" ] +then + ERL_TOP="$1" + shift +fi + +if [ -z $ERL_TOP ] +then + ERL_TOP=`git rev-parse --show-toplevel` + if [ $? = 0 ] + then + print_highlighted_msg $LIGHT_CYAN "The environment variable ERL_TOP has been set to the git root" + else + echo "The ERL_TOP environment variable need to be set before this script is executed." + exit 1 + fi +fi + +export ERL_TOP=$ERL_TOP + + +if [ -z "${ARGS}" ] +then + ARGS="$@" +fi + +# make test in root +DIR=`pwd` +if [ "$DIR" -ef "$ERL_TOP" ] +then + TARGET_SYS=`$ERL_TOP/erts/autoconf/config.guess` + REL_DIR="$ERL_TOP/release/$TARGET_SYS" + cd "$REL_DIR" + ./Install -minimal "`pwd`" + export PATH="$REL_DIR/bin:$PATH" + cd "$ERL_TOP/release/tests/test_server" + print_all_tests_takes_long_time_warning + echo "The tests will start in a few seconds..." + sleep 45 + cd "$ERL_TOP/release/tests/test_server" + erl -eval "ts:install(),erlang:halt()" + erl -noinput -eval "ts:run([all_tests,batch]),erlang:halt()" + exit $? +fi + +# check that there is a test directory +if [ ! -d test ] +then + print_highlighted_msg $RED "This target only works in directories containing a test directory or\nin the root directory." + exit 1 +fi + + +APPLICATION="`basename $DIR`" +CT_RUN="$ERL_TOP/bin/ct_run" +MAKE_TEST_DIR="`pwd`/make_test_dir" +MAKE_TEST_REL_DIR="$MAKE_TEST_DIR/${APPLICATION}_test" +MAKE_TEST_CT_LOGS="$MAKE_TEST_DIR/ct_logs" +RELEASE_TEST_SPEC_LOG="$MAKE_TEST_CT_LOGS/release_tests_spec_log" + +cd test +echo "The tests in test directory for $APPLICATION will be executed with ct_run" +if [ -z "${ARGS}" ] +then + if [ ! -d "$MAKE_TEST_DIR" ] + then + print_all_tests_for_application_notes + fi + if find . -type f -name '*.c' | grep -q "." + then + print_c_files_warning + fi +fi + +mkdir -p "$MAKE_TEST_DIR" +mkdir -p "$MAKE_TEST_REL_DIR" +mkdir -p "$MAKE_TEST_CT_LOGS" +make RELSYSDIR=$MAKE_TEST_REL_DIR release_tests_spec > $RELEASE_TEST_SPEC_LOG 2>&1 + +if [ $? != 0 ] +then + cat $RELEASE_TEST_SPEC_LOG + print_highlighted_msg $RED "\"make RELSYSDIR="$MAKE_TEST_REL_DIR" release_tests_spec\" failed." + exit 1 +fi +SPEC_FLAG="" +SPEC_FILE="" +if [ -z "${ARGS}" ] +then + SPEC_FLAG="-spec" + SPEC_FILE="$MAKE_TEST_REL_DIR/$APPLICATION.spec" + ARGS="$SPEC_FLAG $SPEC_FILE" +fi +# Compile test server +(cd "$ERL_TOP/lib/common_test/test_server" && make) +# Run ct_run +cd $MAKE_TEST_REL_DIR +$CT_RUN -logdir $MAKE_TEST_CT_LOGS\ + -pa "$ERL_TOP/lib/common_test/test_server"\ + ${ARGS}\ + -erl_args\ + -env "$PATH"\ + -env ERL_CRASH_DUMP "$MAKE_TEST_DIR/${APPLICATION}_erl_crash.dump"\ + -boot start_sasl\ + -sasl errlog_type error\ + -pz "$ERL_TOP/lib/common_test/test_server"\ + -pz "."\ + -ct_test_vars "{net_dir,\"\"}"\ + -noshell\ + -sname test_server\ + -rsh ssh\ + ${ERL_ARGS} +CT_RUN_STATUS=$? +if [ $CT_RUN_STATUS = "0" ] +then + print_highlighted_msg $GREEN "The test(s) ran successfully (ct_run returned a success code)\nTest logs: file://$MAKE_TEST_CT_LOGS/index.html" + exit 0 +else + print_on_error_note + print_highlighted_msg $RED "ct_run returned the error code $CT_RUN_STATUS\nTest logs: file://$MAKE_TEST_CT_LOGS/index.html" + exit $CT_RUN_STATUS +fi diff --git a/scripts/pre-push b/scripts/pre-push index 71e9fd1e75..7da1f575db 100755 --- a/scripts/pre-push +++ b/scripts/pre-push @@ -22,12 +22,15 @@ # <local ref> <local sha1> <remote ref> <remote sha1> # -NEW_RELEASES="21 20 19 18 17" +# Bump this version to give users an update notification. +PRE_PUSH_SCRIPT_VERSION=1 + +NEW_RELEASES="22 21 20 19 18 17" OLD_RELEASES="r16 r15 r14 r13" RELEASES="$NEW_RELEASES $OLD_RELEASES" # First commit on master, not allowed in other branches -MASTER_ONLY=aea2a053e28a11497796879715be29ab0c3cd1a0 +MASTER_ONLY=f633fe962ea7078c32f8c81d34950c0ebce0f472 # Number of commits and files allowed in one push by this script NCOMMITS_MAX=100 @@ -54,13 +57,23 @@ null=0000000000000000000000000000000000000000 #echo "pre-push hook: remote=$remote" #echo "pre-push hook: url=$url" +red_on() { + printf '%b' "\033[31m" +} + +red_off() { + printf '%b' "\033[0m" +} + if [ "$url" = 'https://github.com/erlang/otp.git' -o "$url" = 'git@github.com:erlang/otp.git' ] then if [ $remote = "$url" ]; then + red_on echo "$0 says:" echo "***" echo "*** Push to $url without using a named remote is NOT ALLOWED!!!!" echo "***" + red_off exit 1 fi IFS=' ' @@ -73,18 +86,22 @@ then if [ "$local_sha" = $null ] then + red_on echo "$0 says:" echo "***" echo "*** DELETE push to '$remote' NOT ALLOWED!!!!!" echo "***" + red_off exit 1 fi if [ "$local_ref" != "$remote_ref" ] then + red_on echo "$0 says:" echo "***" echo "*** RENAME push: $local_ref pushed as $remote_ref to '$remote' NOT ALLOWED!!!!" echo "***" + red_off exit 1 fi case "$remote_ref" in @@ -92,46 +109,74 @@ then branch=${remote_ref#refs/heads/} if [ "$remote_sha" = $null ] then + red_on echo "$0 says:" echo "***" echo "*** UNKNOWN BRANCH: '$branch' does not exist at '$remote'!!!!" echo "***" + red_off exit 1 fi if ! git log -1 --oneline $remote_sha > /dev/null 2>&1 then + red_on echo "$0 says:" echo "***" echo "*** The top of '$branch' at '$remote' ($remote_sha)" echo "*** does not exist locally!!!" echo "*** You probably need to refresh local '$branch' and redo merge." echo "***" + red_off exit 1 fi if ! git merge-base --is-ancestor $remote_sha $local_sha then + red_on echo "$0 says:" echo "***" echo "*** FORCE push branch to '$remote' NOT ALLOWED!!!" echo "***" + red_off exit 1 fi if [ $remote_ref != refs/heads/master -a "$MASTER_ONLY" ] && git merge-base --is-ancestor $MASTER_ONLY $local_sha then - echo "$0 says:" - echo "***" - echo "*** INVALID MERGE: Commit $MASTER_ONLY should not be reachable from '$branch'!!!!" - echo "*** You have probably merged master into '$branch' by mistake" - echo "***" - exit 1 + THIS_SCRIPT=`git rev-parse --git-path hooks/pre-push` + THIS_SCRIPT=`realpath $THIS_SCRIPT` + if git show refs/remotes/$remote/master:scripts/pre-push | diff -q --context=0 $THIS_SCRIPT - > /dev/null 2>&1 + then + red_on + echo "$0 says:" + echo "***" + echo "*** INVALID MERGE: Commit $MASTER_ONLY should not be reachable from '$branch'!!!!" + echo "*** You have probably merged master into '$branch' by mistake" + echo "***" + red_off + exit 1 + else + red_on + echo "$0 says:" + echo "***" + echo "*** The pre-push hook of this OTP repo needs updating." + echo "*** Do it by executing the following command:" + echo "***" + echo "*** git show refs/remotes/$remote/master:scripts/pre-push > $THIS_SCRIPT" + echo "***" + echo "*** And then retry the push." + echo "***" + red_off + exit 1 + fi fi if [ ${remote_ref#refs/heads/maint-} != $remote_ref ] && git merge-base --is-ancestor refs/remotes/$remote/maint $local_sha then + red_on echo "$0 says:" echo "***" echo "*** INVALID MERGE: Branch maint should not be reachable from '$branch'!!!!" echo "*** You have probably merged maint into '$branch' by mistake." echo "***" + red_off exit 1 fi if [ $remote_ref = refs/heads/maint -o $remote_ref = refs/heads/master ]; then @@ -147,29 +192,35 @@ then fi if [ $remote_ref = refs/heads/master ] && ! git merge-base --is-ancestor refs/remotes/$remote/maint $local_sha then + red_on echo "$0 says:" echo "***" echo "*** INVALID PUSH: Branch '$remote/maint' is not reachable from master!!!!" echo "*** Someone needs to merge maint forward to master and push." echo "***" + red_off exit 1 fi NCOMMITS=`git rev-list --count $remote_sha..$local_sha` if [ $NCOMMITS -gt $NCOMMITS_MAX ] then + red_on echo "$0 says:" echo "***" echo "*** HUGE push: $NCOMMITS commits (> $NCOMMITS_MAX) to '$branch' at '$remote' NOT ALLOWED!!!!" echo "***" + red_off exit 1 fi NFILES=`git diff --name-only $remote_sha $local_sha | wc --lines` if [ $NFILES -gt $NFILES_MAX ] then + red_on echo "$0 says:" echo "***" echo "*** HUGE push: $NFILES changed files (> $NFILES_MAX) to '$branch' at '$remote' NOT ALLOWED!!!!" echo "***" + red_off exit 1 fi ;; @@ -185,49 +236,74 @@ then done if [ $REL = "UNKNOWN" ] then + red_on echo "$0 says:" echo "***" echo "*** Unknown OTP release number in tag '$tag'" echo "***" + red_off exit 1 fi if [ "$remote_sha" != $null ] then + red_on echo "$0 says:" echo "***" echo "*** FORCE push tag to '$remote' NOT ALLOWED!!!" echo "*** Tag '$tag' already exists at '$remote'." echo "***" + red_off exit 1 fi ;; refs/heads/*) branch=${remote_ref#refs/heads/} + red_on echo "$0 says:" echo "***" echo "*** UNKNOWN branch name: '$branch' pushed to '$remote' NOT ALLOWED!!!!" echo "***" + red_off exit 1 ;; refs/tags/*) tag=${remote_ref#refs/tags/} + red_on echo "$0 says:" echo "***" echo "*** UNKNOWN tag name: '$tag' pushed to '$remote' NOT ALLOWED!!!!" echo "***" + red_off exit 1 ;; *) + red_on echo "$0 says:" echo "***" echo "*** STRANGE ref: '$remote_ref' pushed to '$remote' NOT ALLOWED!!!!" echo "***" + red_off exit 1 ;; esac done + + echo "$0: OK" + + THIS_SCRIPT=`git rev-parse --git-path hooks/pre-push` + THIS_SCRIPT=`realpath $THIS_SCRIPT` + if git show refs/remotes/$remote/master:scripts/pre-push | diff --context=0 $THIS_SCRIPT - | grep -q PRE_PUSH_SCRIPT_VERSION > /dev/null 2>&1 + then + echo "" + echo "NOTE: There is a newer version of the pre-push hook in this OTP repo." + echo " You can install it by executing the following command:" + echo + echo " git show refs/remotes/$remote/master:scripts/pre-push > $THIS_SCRIPT" + echo + fi else echo "$0: No checks done for remote '$remote' at $url." fi exit 0 + |
