diff options
author | Hans Nilsson <hans@erlang.org> | 2020-01-09 14:28:43 +0100 |
---|---|---|
committer | Hans Nilsson <hans@erlang.org> | 2020-01-09 14:28:43 +0100 |
commit | 89ae1229c680c9cfa3c9c61a270ace2a25e7a9b4 (patch) | |
tree | 7d23317d270bd6524738a02913eb676fc4c7d9af /lib/common_test | |
parent | 820cf0755cbf8d27388d29b609b70243b104d466 (diff) | |
parent | 320bdd5870eee3660581866b9319b99dcdd8a0a3 (diff) | |
download | erlang-89ae1229c680c9cfa3c9c61a270ace2a25e7a9b4.tar.gz |
Merge branch 'maint'
* maint:
common_test: Comments from doc review
common_test: Document property test reporting
common_test: Add 'prop_tools' option to Config for ct_property_test
common_test: ct_property_test test case
common_test: Add .hrl for support macros and -include of proptest tool
common_test: Remove warning for experimental code
common_test: Presentation function
Diffstat (limited to 'lib/common_test')
-rw-r--r-- | lib/common_test/doc/src/Makefile | 1 | ||||
-rw-r--r-- | lib/common_test/doc/src/ct_property_test.xml | 240 | ||||
-rw-r--r-- | lib/common_test/doc/src/ct_property_test_chapter.xml | 249 | ||||
-rw-r--r-- | lib/common_test/doc/src/part.xml | 1 | ||||
-rw-r--r-- | lib/common_test/include/ct_property_test.hrl | 40 | ||||
-rw-r--r-- | lib/common_test/src/Makefile | 3 | ||||
-rw-r--r-- | lib/common_test/src/ct_property_test.erl | 312 | ||||
-rw-r--r-- | lib/common_test/test/Makefile | 3 | ||||
-rw-r--r-- | lib/common_test/test/ct_property_test_SUITE.erl | 24 | ||||
-rw-r--r-- | lib/common_test/test/property_test/ct_prop.erl | 18 |
10 files changed, 834 insertions, 57 deletions
diff --git a/lib/common_test/doc/src/Makefile b/lib/common_test/doc/src/Makefile index b5acdc6f95..ae06572752 100644 --- a/lib/common_test/doc/src/Makefile +++ b/lib/common_test/doc/src/Makefile @@ -74,6 +74,7 @@ XML_CHAPTER_FILES = \ ct_master_chapter.xml \ event_handler_chapter.xml \ ct_hooks_chapter.xml \ + ct_property_test_chapter.xml \ dependencies_chapter.xml \ notes.xml diff --git a/lib/common_test/doc/src/ct_property_test.xml b/lib/common_test/doc/src/ct_property_test.xml index 1e01d9a5d7..1690e9962a 100644 --- a/lib/common_test/doc/src/ct_property_test.xml +++ b/lib/common_test/doc/src/ct_property_test.xml @@ -33,30 +33,39 @@ <file>ct_property_test.xml</file> </header> <module since="OTP 17.3">ct_property_test</module> - <modulesummary>EXPERIMENTAL support in Common Test for calling - property-based tests.</modulesummary> + <modulesummary>Support in Common Test for running property-based tests.</modulesummary> <description> - <p>EXPERIMENTAL support in <c>Common Test</c> for calling property-based - tests.</p> + <p>This module helps running property-based tests in the + <c>Common Test</c> framework. One (or more) of the property testing tools + </p> + <list> + <item><url href="http://www.quviq.com">QuickCheck</url>,</item> + <item><url href="https://proper-testing.github.io">PropEr</url> or</item> + <item><url href="https://github.com/krestenkrab/triq">Triq</url></item> + </list> + <p> + is assumed to be installed. + </p> - <p>This module is a first step to run property-based tests in the - <c>Common Test</c> framework. A property testing tool like QuickCheck - or PropEr is assumed to be installed.</p> - - <p>The idea is to have a <c>Common Test</c> test suite calling a property - testing tool with special property test suites as defined by that tool. - The usual Erlang application directory structure is assumed. The tests - are collected in the <c>test</c> directory of the application. The - <c>test</c> directory has a subdirectory <c>property_test</c>, where - everything needed for the property tests is collected.</p> + <p>The idea with this module is to have a <c>Common Test</c> test suite calling + a property testing tool with special property test suites as defined by that tool. + The tests + are collected in the <c>test</c> directory of the application. The + <c>test</c> directory has a subdirectory <c>property_test</c>, where + everything needed for the property tests are collected. + The usual Erlang application directory structure is assumed. + </p> <p>A typical <c>Common Test</c> test suite using <c>ct_property_test</c> - is organized as follows:</p> + is organized as follows:</p> + + <code> +-module(my_prop_test_SUITE). +-compile(export_all). - <pre> - -include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct.hrl"). all() -> [prop_ftp_case]. @@ -66,51 +75,198 @@ %%%---- test case prop_ftp_case(Config) -> ct_property_test:quickcheck( - ftp_simple_client_server:prop_ftp(Config), + ftp_simple_client_server:prop_ftp(), Config - ).</pre> + ).</code> + <p>and the the property test module (in this example <c>ftp_simple_client_server.erl</c>) + as almost a usual property testing module + (More examples are in <seealso marker="ct_property_test_chapter">the User's Guide</seealso>):</p> + <code> +-module(ftp_simple_client_server). +-export([prop_ftp/0...]). - <warning> - <p>This is experimental code that can be changed or removed anytime - without any warning.</p> - </warning> +-include_lib("common_test/include/ct_property_test.hrl"). +prop_ftp() -> + ?FORALL( .... + </code> </description> <funcs> <func> <name since="OTP 17.3">init_per_suite(Config) -> Config | {skip, Reason}</name> - <fsummary>Initializes Config for property testing.</fsummary> + <fsummary>Initializes and extends Config for property testing.</fsummary> <desc><marker id="init_per_suite-1"/> - <p>Initializes <c>Config</c> for property testing.</p> + <p>Initializes and extends <c>Config</c> for property based testing.</p> <p>This function investigates if support is available for either - Quickcheck, PropEr, or Triq. The options - <c>{property_dir,AbsPath}</c> and <c>{property_test_tool,Tool}</c> - are set in the <c>Config</c> returned.</p> + <url href="http://www.quviq.com">QuickCheck</url>, + <url href="https://proper-testing.github.io">PropEr</url> + or <url href="https://github.com/krestenkrab/triq">Triq</url> and compiles the + properties with the first tool found. + It is supposed to be called in the <c>init_per_suite/1</c> function + in a CommonTest test suite. + </p> + <p>Which tools to check for, and in which order could be set with the option + <c>{prop_tools, list(eqc|proper|triq)}</c> + in the CommonTest configuration <c>Config</c>. The default value is + <c>[eqc, proper, triq]</c> with <c>eqc</c> being the first one searched for. + </p> + <p>If no support is found for any tool, this function returns + <c>{skip, Explanation}</c>. + </p> + <p>If support is found, the option <c>{property_test_tool,ToolModule}</c> with + the selected tool main module name (<c>eqc</c>, <c>proper</c> or <c>triq</c>) + is added to the list <c>Config</c> which then is returned. + </p> + <p>The property tests are assumed to be in a subdirectory named + <c>property_test</c>. + All found Erlang files in that directory are compiled with one of the macros + <c>'EQC'</c>, <c>'PROPER'</c> or <c>'TRIQ'</c> set, depending on which tool + that is first found. This could make parts of the Erlang property tests + code to be included or excluded with the macro directives + <c>-ifdef(Macro).</c> or <c>-ifndef(Macro).</c>. + </p> + <p>The file(s) in the <c>property_test</c> subdirectory could, or should, + include the ct_property_test include file: + </p> + <code> +-include_lib("common_test/include/ct_property_test.hrl"). + </code> + <p>This included file will: + </p> + <list> + <item>Include the correct tool's include file</item> + <item>Set the macro <c>'MOD_eqc'</c> to the correct module name for the + selected tool. That is, the macro <c>'MOD_eqc'</c> is set to either + <c>eqc</c>, <c>proper</c> or <c>triq</c>. + </item> + </list> + </desc> + </func> - <p>The function is intended to be called in function - <c>init_per_suite</c> in the test suite.</p> + <func> + <name since="OTP 17.3">quickcheck(Property, Config) -> true | {fail, Reason}</name> + <fsummary>Calls quickcheck and returns the result in a form suitable for + Common Test.</fsummary> + <desc> + <p>Calls the selected tool's function for running the <c>Property</c>. It is usually and + by historical reasons called quickcheck, and that is why that name is used in + this module (<c>ct_property_test</c>). + </p> + <p>The result is returned in a form suitable for <c>Common Test</c> test suites. + </p> + <p>This function is intended to be called in test cases in test suites. + </p> + </desc> + </func> - <p>The property tests are assumed to be in subdirectory - <c>property_test</c>.</p> + <func> + <name since="">present_result(Module, Cmds, Triple, Config) -> Result</name> + <fsummary>Presents the result of statem property testing</fsummary> + <desc> + <p>Same as <seealso marker="#present_result/5"><c>present_result(Module, Cmds, Triple, Config, [])</c></seealso> + </p> </desc> </func> <func> - <name since="OTP 17.3">quickcheck(Property, Config) -> true | {fail, Reason}</name> - <fsummary>Calls quickcheck and returns the result in a form suitable for - Common Test.</fsummary> - <desc><marker id="quickcheck-2"/> - <p>Calls quickcheck and returns the result in a form suitable for - <c>Common Test</c>.</p> + <name since="">present_result(Module, Cmds, Triple, Config, Options) -> Result</name> + <fsummary>Presents the result of statem property testing</fsummary> + <type> + <v>Module = module()</v> + <d></d> + + <v>Cmds =</v> + <d>the list of commands generated by the property testing tool, for example + by proper:commands/1 or by proper:parallel_commands/1 + </d> + + <v>Triple =</v> + <d>the output from for example proper:run_commands/2 or proper:run_parallel_commands/2</d> + + <v>Config =</v> + <d>the Common Test <seealso marker="common_test#Module:Testcase/1">Config</seealso> in test cases.</d> + + <v>Options = [present_option()]</v> + <v>present_option() = {print_fun, fun(Format,Args)}</v> + <v> | {spec, StatisticsSpec}</v> + <d>The <c>print_fun</c> defines which function to do the actual printout. The default is + <seealso marker="ct#log/2">ct:log/2</seealso>. + The <c>spec</c> defines what statistics are to be printed<!--, see the + <seealso marker="ct_property_test_chapter#spec_present_result">User's Guide</seealso>--> + </d> + + <v>Result = boolean()</v> + <d>Is <c>false</c> if the test failed and is <c>true</c> if the test passed</d> + </type> + <desc> + <p>Presents the result of <i>stateful (statem) property testing</i> using the aggregate function in + PropEr, QuickCheck or other similar property testing tool. + </p> + <p>It is assumed to be called inside the property called by + <seealso marker="#quickcheck/2">quickcheck/2</seealso>:</p> + <code> +... +RunResult = run_parallel_commands(?MODULE, Cmds), +ct_property_test:present_result(?MODULE, Cmds, RunResult, Config) +... + </code> + <p>See the <seealso marker="ct_property_test_chapter#stateful1">User's Guide</seealso> for + an example of the usage and of the default printout. + </p> + <p>The <c>StatisticsSpec</c> is a list of the tuples:</p> + <list> + <item><c>{Title::string(), CollectFun::fun/1}</c></item> + <item><c>{Title::string(), FrequencyFun::/0, CollectFun::fun/1}</c></item> + </list> + <p>Each tuple will produce one table in the order of their places in the list.</p> + <list> + <item><c>Title</c> will be the title of one result table</item> - <p>This function is intended to be called in the test cases in the - test suite.</p> + <item><c>CollectFun</c> is called with one argument: the <c>Cmds</c>. It should return + a list of the values to be counted. The following pre-defined functions exist: + <list> + <item><c>ct_property_test:cmnd_names/1</c> returns a list of commands (function calls) generated in the <c>Cmnd</c> + sequence, without Module, Arguments and other details.</item> + <item><c>ct_property_test:num_calls/1</c> returns a list of the length of commands lists</item> + <item><c>ct_property_test:sequential_parallel/1</c> returns a list with information about sequential and + parallel parts from <c>Tool:parallel_commands/1,2</c></item> + </list> + </item> + + <item><c>FrequencyFun/0</c> returns a fun/1 which is supposed to take a list of items as input, + and return an iolist wich will be printed as the table. Per default, the number of each item is counted + and the percentage is printed for each. The list [a,b,a,a,c] could for example return + <pre> + ["a 60%\n","b 20%\n","c 20%\n"]</pre> + which will be printed by the <c>print_fun</c>. + The default <c>print_fun</c> will print it as: + <pre> + a 60% + b 20% + c 20%</pre> + </item> + </list> + <p>The default <c>StatisticsSpec</c> is:</p> + <list> + <item>For sequential commands: + <code> +[{"Function calls", fun cmnd_names/1}, + {"Length of command sequences", fun print_frequency_ranges/0, + fun num_calls/1}] + </code></item> + <item>For parallel commands: + <code> +[{"Distribution sequential/parallel", fun sequential_parallel/1}, + {"Function calls", fun cmnd_names/1}, + {"Length of command sequences", fun print_frequency_ranges/0, + fun num_calls/1}] + </code></item> + </list> </desc> </func> + </funcs> </erlref> - - diff --git a/lib/common_test/doc/src/ct_property_test_chapter.xml b/lib/common_test/doc/src/ct_property_test_chapter.xml new file mode 100644 index 0000000000..131f3a962d --- /dev/null +++ b/lib/common_test/doc/src/ct_property_test_chapter.xml @@ -0,0 +1,249 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!DOCTYPE chapter SYSTEM "chapter.dtd"> + +<chapter> + <header> + <copyright> + <year>2019</year><year>2019</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>Common Test's Property Testing Support: ct_property_test</title> + <prepared>Hans Nilsson</prepared> + <docno></docno> + <date></date> + <rev></rev> + <file>ct_property_test_chapter.xml</file> + </header> + + <section> + <marker id="general"></marker> + <title>General</title> + <p> + The <em>Common Test Property Testing Support (ct_property_test)</em> + is an aid to run property based testing tools in Common Test test suites. + </p> + <p> + Basic knowledge of property based testing is assumed in the following. + It is also assumed that at least one of the following property based + testing tools is installed and available in the library path: + </p> + <list> + <item><url href="http://www.quviq.com">QuickCheck</url>,</item> + <item><url href="https://proper-testing.github.io">PropEr</url> or</item> + <item><url href="https://github.com/krestenkrab/triq">Triq</url></item> + </list> + </section> + + <section> + <marker id="supported"></marker> + <title>What Is Supported?</title> + <p>The <seealso marker="ct_property_test#">ct_property_test</seealso> module + does the following: + </p> + <list type="bulleted"> + <item>Compiles the files with property tests in the subdirectory <c>property_test</c> + </item> + <item>Tests properties in those files using the first found Property Testing Tool. + </item> + <item>Saves the results - that is the printouts - in the usual Common Test Log + </item> + </list> + </section> + + + <section> + <title>Introductory Example</title> + <p>Assume that we want to test the lists:sort/1 function. + </p> + <p>We need a property to test the function. In normal way, we create + <c>property_test/ct_prop.erl</c> module in the <c>test</c> directory + in our application: + </p> + + <code> +-module(ct_prop). +-export([prop_sort/0]). + +%%% This will include the .hrl file for the installed testing tool: +-include_lib("common_test/include/ct_property_test.hrl"). + +%%% The property we want to check: +%%% For all possibly unsorted lists, +%%% the result of lists:sort/1 is sorted. +prop_sort() -> + ?FORALL(UnSorted, list(), + is_sorted(lists:sort(UnSorted)) + ). + +%%% Function to check that a list is sorted: +is_sorted([]) -> + true; +is_sorted([_]) -> + true; +is_sorted([H1,H2|SortedTail]) when H1 =< H2 -> + is_sorted([H2|SortedTail]); +is_sorted(_) -> + false. + </code> + + <p>We also need a CommonTest test suite: + </p> + <code> +-module(ct_property_test_SUITE). +-compile(export_all). % Only in tests! + +-include_lib("common_test/include/ct.hrl"). + +all() -> [prop_sort + ]. + +%%% First prepare Config and compile the property tests for the found tool: +init_per_suite(Config) -> + ct_property_test:init_per_suite(Config). + +end_per_suite(Config) -> + Config. + +%%%================================================================ +%%% Test suites +%%% +prop_sort(Config) -> + ct_property_test:quickcheck( + ct_prop:prop_sort(), + Config + ). + </code> + + <p>We run it as usual, for example with ct_run in the OS shell:</p> + <pre> +..../test$ ct_run -suite ct_property_test_SUITE +..... +Common Test: Running make in test directories... + +TEST INFO: 1 test(s), 1 case(s) in 1 suite(s) + +Testing lib.common_test.ct_property_test_SUITE: Starting test, 1 test cases + +---------------------------------------------------- +2019-12-18 10:44:46.293 +Found property tester proper +at "/home/X/lib/proper/ebin/proper.beam" + + +---------------------------------------------------- +2019-12-18 10:44:46.294 +Compiling in "/home/..../test/property_test" + Deleted: ["ct_prop.beam"] + ErlFiles: ["ct_prop.erl"] + MacroDefs: [{d,'PROPER'}] + +Testing lib.common_test.ct_property_test_SUITE: TEST COMPLETE, 1 ok, 0 failed of 1 test cases + +.... + </pre> + </section> + + + <section> + <marker id="stateful1"></marker> + <title>A stateful testing example</title> + <p>Assume a test that generates some parallel stateful commands, and runs 300 tests:</p> + <code> +prop_parallel(Config) -> + numtests(300, + ?FORALL(Cmds, parallel_commands(?MODULE), + begin + RunResult = run_parallel_commands(?MODULE, Cmds), + ct_property_test:present_result(?MODULE, Cmds, RunResult, Config) + end)). + </code> + <p>The + <seealso marker="ct_property_test#present_result/4">ct_property_test:present_result/4</seealso> + is a help function for printing some statistics in the CommonTest log file.</p> + <p>Our example test could for example be a simple test of an ftp server, where we perform get, put + and delete requests, some of them in parallel. Per default, the result has three sections: + </p> + <pre> +*** User 2019-12-11 13:28:17.504 *** + +Distribution sequential/parallel + + 57.7% sequential + 28.0% parallel_2 + 14.3% parallel_1 + + + +*** User 2019-12-11 13:28:17.505 *** + +Function calls + + 44.4% get + 39.3% put + 16.3% delete + + + +*** User 2019-12-11 13:28:17.505 *** + +Length of command sequences + +Range : Number in range +-------:---------------- + 0 - 4: 8 2.7% <-- min=3 + 5 - 9: 44 14.7% +10 - 14: 74 24.7% +15 - 19: 60 20.0% <-- mean=18.7 <-- median=16.0 +20 - 24: 38 12.7% +25 - 29: 26 8.7% +30 - 34: 19 6.3% +35 - 39: 19 6.3% +40 - 44: 8 2.7% +45 - 49: 4 1.3% <-- max=47 + ------ + 300 + </pre> + <p>The first part - <i>Distribution sequential/parallel</i> - shows the distribution in the + sequential and parallel part of the result of parallel_commands/1. See any property testing tool for + an explanation of this function. + The table shows that of all commands (get and put in our case), + 57.7% are executed in the sequential part prior to the parallel part, + 28.0% are executed in the first parallel list and the rest in the second parallel list. + </p> + + <p>The second part - <i>Function calls</i> - shows the distribution of the three calls in the + generated command lists. We see that all of the three calls are executed. If it was so that we + thought that we also generated a fourth call, a table like this shows that we failed with that. + </p> + + <p>The third and final part - <i>Length of command sequences</i> - show statistics of the + generated command sequences. We see that the shortest list has three elementes while the longest + has 47 elements. The mean and median values are also shown. Further we could for example see that + only 2.7% of the lists (that is eight lists) only has three or four elements. + </p> + + </section> + + <!--section> + <marker id="spec_present_result"></marker> + <title>The spec for present_result/5</title> + <p>To be written... + <seealso marker="ct_property_test#present_result/5">present_result/5</seealso> + </p> + </section--> +</chapter> diff --git a/lib/common_test/doc/src/part.xml b/lib/common_test/doc/src/part.xml index 000eb06b82..66dcf75258 100644 --- a/lib/common_test/doc/src/part.xml +++ b/lib/common_test/doc/src/part.xml @@ -48,6 +48,7 @@ <xi:include href="dependencies_chapter.xml"/> <xi:include href="ct_hooks_chapter.xml"/> <xi:include href="why_test_chapter.xml"/> + <xi:include href="ct_property_test_chapter.xml"/> </part> diff --git a/lib/common_test/include/ct_property_test.hrl b/lib/common_test/include/ct_property_test.hrl new file mode 100644 index 0000000000..9d5933fde3 --- /dev/null +++ b/lib/common_test/include/ct_property_test.hrl @@ -0,0 +1,40 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2004-2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +%% + +-ifndef(CT_PROPERTY_TEST_HRL). + -define(CT_PROPERTY_TEST_HRL, true). + + -ifdef(EQC). + -define(MOD_eqc, eqc). + -include_lib("eqc/include/eqc.hrl"). + -else. + -ifdef(PROPER). + -define(MOD_eqc, proper). + -include_lib("proper/include/proper.hrl"). + -else. + -ifdef(TRIQ). + -define(MOD_eqc, triq). + -include_lib("triq/include/triq.hrl"). + -endif. + -endif. + -endif. + +-endif. diff --git a/lib/common_test/src/Makefile b/lib/common_test/src/Makefile index 76689dab8c..ffdef8ec39 100644 --- a/lib/common_test/src/Makefile +++ b/lib/common_test/src/Makefile @@ -96,7 +96,8 @@ HRL_FILES = \ ct_netconfc.hrl EXTERNAL_HRL_FILES = \ ../include/ct.hrl \ - ../include/ct_event.hrl + ../include/ct_event.hrl \ + ../include/ct_property_test.hrl EXTERNAL_INC_PATH = ../include diff --git a/lib/common_test/src/ct_property_test.erl b/lib/common_test/src/ct_property_test.erl index 1bcae300a8..b8bb890add 100644 --- a/lib/common_test/src/ct_property_test.erl +++ b/lib/common_test/src/ct_property_test.erl @@ -18,25 +18,37 @@ %% %CopyrightEnd% %% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%%% %%% -%%% WARNING %%% -%%% %%% -%%% This is experimental code which may be changed or removed %%% -%%% anytime without any warning. %%% -%%% %%% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - -module(ct_property_test). -%% API +%%% API +%% Main functions -export([init_per_suite/1, init_tool/1, quickcheck/2]). +%% Result presentation +-export([present_result/4, present_result/5, + title/2, title/3, + sequential_parallel/1, + cmnd_names/1, + num_calls/1, + print_frequency_ranges/0, + print_frequency/0 + ]). + +%%% Mandatory include -include_lib("common_test/include/ct.hrl"). +%%%================================================================ +%%% +%%% API +%%% + +%%%---------------------------------------------------------------- +%%% +%%% Search for a property tester in the lib path, and if found, compile +%%% the property tests +%%% init_per_suite(Config) -> case init_tool(Config) of {skip, _}=Skip -> @@ -55,8 +67,9 @@ init_per_suite(Config) -> end. init_tool(Config) -> - case which_module_exists([eqc,proper,triq]) of - {ok, ToolModule} -> + ToolsToCheck = proplists:get_value(prop_tools, Config, [eqc,proper,triq]), + case which_module_exists(ToolsToCheck) of + {ok,ToolModule} -> case code:where_is_file(lists:concat([ToolModule,".beam"])) of non_existing -> ct:log("Found ~p, but ~tp~n is not found", @@ -73,12 +86,71 @@ init_tool(Config) -> {skip, "No property testing tool found"} end. +%%%---------------------------------------------------------------- +%%% +%%% Call the found property tester (if any) +%%% quickcheck(Property, Config) -> Tool = proplists:get_value(property_test_tool,Config), F = function_name(quickcheck, Tool), mk_ct_return( Tool:F(Property), Tool ). +%%%---------------------------------------------------------------- +%%% +%%% Present a nice table of the statem result +%%% +present_result(Module, Cmds, Triple, Config) -> + present_result(Module, Cmds, Triple, Config, []). + +present_result(Module, Cmds, {H,Sf,Result}, Config, Options0) -> + DefSpec = + if + is_tuple(Cmds) -> + [{"Distribution sequential/parallel", fun sequential_parallel/1}]; + is_list(Cmds) -> + [] + end + ++ [{"Function calls", fun cmnd_names/1}, + {"Length of command sequences", fun print_frequency_ranges/0, fun num_calls/1} + ], + Options = add_default_options(Options0, + [{print_fun, fun ct:log/2}, + {spec, DefSpec} + ]), + do_present_result(Module, Cmds, H, Sf, Result, Config, Options). + + +title(Str, Fun) -> + title(Str, Fun, fun io:format/2). + +title(Str, Fun, PrintFun) -> + fun(L) -> PrintFun("~n~s~n~n~s~n", [Str,Fun(L)]) end. + +print_frequency() -> + fun(L) -> + [io_lib:format("~5.1f% ~p~n",[Pcnt,V]) + || {V,_Num,Pcnt} <- + with_percentage(get_frequencies_no_range(L), length(L)) + ] + end. + +print_frequency_ranges() -> + print_frequency_ranges([{ngroups,10}]). + +print_frequency_ranges(Options0) -> + fun([]) -> + io_lib:format('Empty list!~n',[]); + (L ) -> + try + Options = set_default_print_freq_range_opts(Options0, L), + do_print_frequency_ranges(L, Options) + catch + C:E:S -> + ct:pal("~p:~p ~p:~p~n~p~n~p",[?MODULE,?LINE,C,E,S,L]) + end + end. + %%%================================================================ %%% %%% Local functions @@ -163,3 +235,217 @@ macro_def(triq) -> [{d, 'TRIQ'}]. function_name(quickcheck, triq) -> check; function_name(F, _) -> F. + +%%%================================================================ +%%%================================================================ +%%%================================================================ +%%% +%%% Result presentation part +%%% +do_present_result(_Module, Cmds, _H, _Sf, ok, Config, Options) -> + [PrintFun, Spec] = [proplists:get_value(K,Options) || K <- [print_fun,spec]], + Tool = proplists:get_value(property_test_tool,Config), + AGGREGATE = function_name(aggregate, Tool), + lists:foldr(fun({Title, FreqFun, CollecFun}, Result) -> + Tool:AGGREGATE(title(Title, FreqFun(), PrintFun), + CollecFun(Cmds), + Result); + ({Title, CollecFun}, Result) -> + Tool:AGGREGATE(title(Title, print_frequency(), PrintFun), + CollecFun(Cmds), + Result) + end, true, Spec); + +do_present_result(Module, Cmds, H, Sf, Result, _Config, Options) -> + [PrintFun] = [proplists:get_value(K,Options) || K <- [print_fun]], + PrintFun("Module = ~p,~n" + "Commands = ~p,~n" + "History = ~p,~n" + "FinalDynState = ~p,~n" + "Result = ~p", + [Module, Cmds, H, Sf, Result]), + Result == ok. % Proper dislikes non-boolean results while eqc treats non-true as false. + +%%%================================================================ +cmnd_names(Cs) -> traverse_commands(fun cmnd_name/1, Cs). +cmnd_name(L) -> [F || {set,_Var,{call,_Mod,F,_As}} <- L]. + +num_calls(Cs) -> traverse_commands(fun num_call/1, Cs). +num_call(L) -> [length(L)]. + +sequential_parallel(Cs) -> + traverse_commands(fun(L) -> dup_module(L, sequential) end, + fun(L) -> [dup_module(L1, mkmod("parallel",num(L1,L))) || L1<-L] end, + Cs). +dup_module(L, ModName) -> lists:duplicate(length(L), ModName). +mkmod(PfxStr,N) -> list_to_atom(PfxStr++"_"++integer_to_list(N)). + +%% Meta functions for the aggregate functions +traverse_commands(Fun, L) when is_list(L) -> Fun(L); +traverse_commands(Fun, {Seq, ParLs}) -> Fun(lists:append([Seq|ParLs])). + +traverse_commands(Fseq, _Fpar, L) when is_list(L) -> Fseq(L); +traverse_commands(Fseq, Fpar, {Seq, ParLs}) -> lists:append([Fseq(Seq)|Fpar(ParLs)]). + +%%%================================================================ +-define(middle_dot, 0183). + +set_default_print_freq_range_opts(Opts0, L) -> + add_default_options(Opts0, [{ngroups, 10}, + {min, 0}, + {max, max_in_list(L)} + ]). + +add_default_options(Opts0, DefaultOpts) -> + [set_def_opt(Key,DefVal,Opts0) || {Key,DefVal} <- DefaultOpts]. + +set_def_opt(Key, DefaultValue, Opts) -> + {Key, proplists:get_value(Key, Opts, DefaultValue)}. + +max_in_list(L) -> + case lists:last(L) of + Max when is_integer(Max) -> Max; + {Max,_} -> Max + end. + +do_print_frequency_ranges(L0, Options) -> + [N,Min,Max] = [proplists:get_value(K,Options) || K <- [ngroups, min, max]], + L = if + N>Max -> + %% There will be less than the demanded number of classes, + %% insert one last with zero values in it. That will force + %% the generation of N classes. + L0++[{N,0}]; + N=<Max -> + L0 + end, + try + Interval = round((Max-Min)/N), + IntervalLowerLimits = lists:seq(Min,Max,Interval), + Ranges = [{I,I+Interval-1} || I <- IntervalLowerLimits], + Acc0 = [{Rng,0} || Rng <- Ranges], + Fs0 = get_frequencies(L, Acc0), + SumVal = lists:sum([V||{_,V}<-Fs0]), + Fs = with_percentage(Fs0, SumVal), + DistInfo = [{"min", lists:min(L)}, + {"mean", mean(L)}, + {"median", median(L)}, + {"max", lists:max(L)}], + + Npos_value = num_digits(SumVal), + Npos_range = num_digits(Max), + [%% Table heading: + io_lib:format("Range~*s: ~s~n",[2*Npos_range-2,"", "Number in range"]), + %% Line under heading: + io_lib:format("~*c:~*c~n",[2*Npos_range+3,$-, max(16,Npos_value+10),$- ]), + %% Lines with values: + [io_lib:format("~*w - ~*w: ~*w ~5.1f% ~s~n", + [Npos_range,Rlow, + Npos_range,Rhigh, + Npos_value,Val, + Percent, + cond_prt_vals(DistInfo, Interv) + ]) + || {Interv={Rlow,Rhigh},Val,Percent} <- Fs], + %% Line under the table for the total number of values: + io_lib:format('~*c ~*c~n',[2*Npos_range,32, Npos_value+3,$-]), + %% The total number of values: + io_lib:format('~*c ~*w~n',[2*Npos_range,32, Npos_value,SumVal]) + ] + catch + C:E -> + ct:pal('*** Failed printing (~p:~p) for~n~p~n',[C,E,L]) + end. + +cond_prt_vals(LVs, CurrentInterval) -> + [prt_val(Label, Value, CurrentInterval) || {Label,Value} <- LVs]. + +prt_val(Label, Value, CurrentInterval) -> + case in_interval(Value, CurrentInterval) of + true -> + io_lib:format(" <-- ~s=" ++ if + is_float(Value) -> "~.1f"; + true -> "~p" + end, + [Label,Value]); + false -> + "" + end. + +get_frequencies([{I,Num}|T], [{{Lower,Upper},Cnt}|Acc]) when Lower=<I,I=<Upper -> + get_frequencies(T, [{{Lower,Upper},Cnt+Num}|Acc]); +get_frequencies(L=[{I,_Num}|_], [Ah={{_Lower,Upper},_Cnt}|Acc]) when I>Upper -> + [Ah | get_frequencies(L,Acc)]; +get_frequencies([I|T], Acc) when is_integer(I) -> + get_frequencies([{I,1}|T], Acc); +get_frequencies([], Acc) -> + Acc. + +get_frequencies_no_range([]) -> + io_lib:format("No values~n", []); +get_frequencies_no_range(L) -> + [H|T] = lists:sort(L), + get_frequencies_no_range(T, H, 1, []). + +get_frequencies_no_range([H|T], H, N, Acc) -> + get_frequencies_no_range(T, H, N+1, Acc); +get_frequencies_no_range([H1|T], H, N, Acc) -> + get_frequencies_no_range(T, H1, 1, [{H,N}|Acc]); +get_frequencies_no_range([], H, N, Acc) -> + lists:reverse( + lists:keysort(2, [{H,N}|Acc])). + +%% get_frequencies_percent(L) -> +%% with_percentage(get_frequencies_no_range(L), length(L)). + + +with_percentage(Fs, Sum) -> + [{Rng,Val,100*Val/Sum} || {Rng,Val} <- Fs]. + + +num_digits(I) -> 1+trunc(math:log(I)/math:log(10)). + +num(Elem, List) -> length(lists:takewhile(fun(E) -> E /= Elem end, List)) + 1. + +%%%---- Just for naming an operation for readability +is_odd(I) -> (I rem 2) == 1. + +in_interval(Value, {Rlow,Rhigh}) -> + try + Rlow=<round(Value) andalso round(Value)=<Rhigh + catch + _:_ -> false + end. + +%%%================================================================ +%%% Statistical functions + +%%%---- Mean value +mean(L = [X|_]) when is_number(X) -> + lists:sum(L) / length(L); +mean(L = [{_Value,_Weight}|_]) -> + SumOfWeights = lists:sum([W||{_,W}<-L]), + WeightedSum = lists:sum([W*V||{V,W}<-L]), + WeightedSum / SumOfWeights; +mean(_) -> + undefined. + +%%%---- Median +median(L = [X|_]) when is_number(X) -> + Len = length(L), + case is_odd(Len) of + true -> + hd(lists:nthtail(Len div 2, L)); + false -> + %% 1) L has at least one element (the one in the is_number test). + %% 2) Length is even. + %% => Length >= 2 + [M1,M2|_] = lists:nthtail((Len div 2)-1, L), + (M1+M2) / 2 + end; +%% integer Weights... +median(L = [{_Value,_Weight}|_]) -> + median( lists:append([lists:duplicate(W,V) || {V,W} <- L]) ); +median(_) -> + undefined. + diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile index e510b74d6a..fae7ce0eb5 100644 --- a/lib/common_test/test/Makefile +++ b/lib/common_test/test/Makefile @@ -76,7 +76,8 @@ MODULES= \ ct_unicode_SUITE \ ct_auto_clean_SUITE \ ct_util_SUITE \ - ct_tc_repeat_SUITE + ct_tc_repeat_SUITE \ + ct_property_test_SUITE ERL_FILES= $(MODULES:%=%.erl) HRL_FILES= test_server_test_lib.hrl diff --git a/lib/common_test/test/ct_property_test_SUITE.erl b/lib/common_test/test/ct_property_test_SUITE.erl new file mode 100644 index 0000000000..1f8c9e08cf --- /dev/null +++ b/lib/common_test/test/ct_property_test_SUITE.erl @@ -0,0 +1,24 @@ +-module(ct_property_test_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +all() -> [prop_sort + ]. + +%%% First prepare Config and compile the property tests for the found tool: +init_per_suite(Config) -> + ct_property_test:init_per_suite(Config). + +end_per_suite(Config) -> + Config. + +%%%================================================================ +%%% Test suites +%%% +prop_sort(Config) -> + ct_property_test:quickcheck( + ct_prop:prop_sort(), + Config + ). diff --git a/lib/common_test/test/property_test/ct_prop.erl b/lib/common_test/test/property_test/ct_prop.erl new file mode 100644 index 0000000000..67ab3f3e6b --- /dev/null +++ b/lib/common_test/test/property_test/ct_prop.erl @@ -0,0 +1,18 @@ +-module(ct_prop). +-export([prop_sort/0]). + +-include_lib("common_test/include/ct_property_test.hrl"). + +prop_sort() -> + ?FORALL(UnSorted, list(), + is_sorted(lists:sort(UnSorted)) + ). + +is_sorted([]) -> + true; +is_sorted([_]) -> + true; +is_sorted([H1,H2|SortedTail]) when H1 =< H2 -> + is_sorted([H2|SortedTail]); +is_sorted(_) -> + false. |