summaryrefslogtreecommitdiff
path: root/lib/common_test
diff options
context:
space:
mode:
authorHans Nilsson <hans@erlang.org>2020-01-09 14:28:43 +0100
committerHans Nilsson <hans@erlang.org>2020-01-09 14:28:43 +0100
commit89ae1229c680c9cfa3c9c61a270ace2a25e7a9b4 (patch)
tree7d23317d270bd6524738a02913eb676fc4c7d9af /lib/common_test
parent820cf0755cbf8d27388d29b609b70243b104d466 (diff)
parent320bdd5870eee3660581866b9319b99dcdd8a0a3 (diff)
downloaderlang-89ae1229c680c9cfa3c9c61a270ace2a25e7a9b4.tar.gz
Merge branch 'maint'
* maint: common_test: Comments from doc review common_test: Document property test reporting common_test: Add 'prop_tools' option to Config for ct_property_test common_test: ct_property_test test case common_test: Add .hrl for support macros and -include of proptest tool common_test: Remove warning for experimental code common_test: Presentation function
Diffstat (limited to 'lib/common_test')
-rw-r--r--lib/common_test/doc/src/Makefile1
-rw-r--r--lib/common_test/doc/src/ct_property_test.xml240
-rw-r--r--lib/common_test/doc/src/ct_property_test_chapter.xml249
-rw-r--r--lib/common_test/doc/src/part.xml1
-rw-r--r--lib/common_test/include/ct_property_test.hrl40
-rw-r--r--lib/common_test/src/Makefile3
-rw-r--r--lib/common_test/src/ct_property_test.erl312
-rw-r--r--lib/common_test/test/Makefile3
-rw-r--r--lib/common_test/test/ct_property_test_SUITE.erl24
-rw-r--r--lib/common_test/test/property_test/ct_prop.erl18
10 files changed, 834 insertions, 57 deletions
diff --git a/lib/common_test/doc/src/Makefile b/lib/common_test/doc/src/Makefile
index b5acdc6f95..ae06572752 100644
--- a/lib/common_test/doc/src/Makefile
+++ b/lib/common_test/doc/src/Makefile
@@ -74,6 +74,7 @@ XML_CHAPTER_FILES = \
ct_master_chapter.xml \
event_handler_chapter.xml \
ct_hooks_chapter.xml \
+ ct_property_test_chapter.xml \
dependencies_chapter.xml \
notes.xml
diff --git a/lib/common_test/doc/src/ct_property_test.xml b/lib/common_test/doc/src/ct_property_test.xml
index 1e01d9a5d7..1690e9962a 100644
--- a/lib/common_test/doc/src/ct_property_test.xml
+++ b/lib/common_test/doc/src/ct_property_test.xml
@@ -33,30 +33,39 @@
<file>ct_property_test.xml</file>
</header>
<module since="OTP 17.3">ct_property_test</module>
- <modulesummary>EXPERIMENTAL support in Common Test for calling
- property-based tests.</modulesummary>
+ <modulesummary>Support in Common Test for running property-based tests.</modulesummary>
<description>
- <p>EXPERIMENTAL support in <c>Common Test</c> for calling property-based
- tests.</p>
+ <p>This module helps running property-based tests in the
+ <c>Common Test</c> framework. One (or more) of the property testing tools
+ </p>
+ <list>
+ <item><url href="http://www.quviq.com">QuickCheck</url>,</item>
+ <item><url href="https://proper-testing.github.io">PropEr</url> or</item>
+ <item><url href="https://github.com/krestenkrab/triq">Triq</url></item>
+ </list>
+ <p>
+ is assumed to be installed.
+ </p>
- <p>This module is a first step to run property-based tests in the
- <c>Common Test</c> framework. A property testing tool like QuickCheck
- or PropEr is assumed to be installed.</p>
-
- <p>The idea is to have a <c>Common Test</c> test suite calling a property
- testing tool with special property test suites as defined by that tool.
- The usual Erlang application directory structure is assumed. The tests
- are collected in the <c>test</c> directory of the application. The
- <c>test</c> directory has a subdirectory <c>property_test</c>, where
- everything needed for the property tests is collected.</p>
+ <p>The idea with this module is to have a <c>Common Test</c> test suite calling
+ a property testing tool with special property test suites as defined by that tool.
+ The tests
+ are collected in the <c>test</c> directory of the application. The
+ <c>test</c> directory has a subdirectory <c>property_test</c>, where
+ everything needed for the property tests are collected.
+ The usual Erlang application directory structure is assumed.
+ </p>
<p>A typical <c>Common Test</c> test suite using <c>ct_property_test</c>
- is organized as follows:</p>
+ is organized as follows:</p>
+
+ <code>
+-module(my_prop_test_SUITE).
+-compile(export_all).
- <pre>
- -include_lib("common_test/include/ct.hrl").
+-include_lib("common_test/include/ct.hrl").
all() -&gt; [prop_ftp_case].
@@ -66,51 +75,198 @@
%%%---- test case
prop_ftp_case(Config) -&gt;
ct_property_test:quickcheck(
- ftp_simple_client_server:prop_ftp(Config),
+ ftp_simple_client_server:prop_ftp(),
Config
- ).</pre>
+ ).</code>
+ <p>and the the property test module (in this example <c>ftp_simple_client_server.erl</c>)
+ as almost a usual property testing module
+ (More examples are in <seealso marker="ct_property_test_chapter">the User's Guide</seealso>):</p>
+ <code>
+-module(ftp_simple_client_server).
+-export([prop_ftp/0...]).
- <warning>
- <p>This is experimental code that can be changed or removed anytime
- without any warning.</p>
- </warning>
+-include_lib("common_test/include/ct_property_test.hrl").
+prop_ftp() -&gt;
+ ?FORALL( ....
+ </code>
</description>
<funcs>
<func>
<name since="OTP 17.3">init_per_suite(Config) -&gt; Config | {skip, Reason}</name>
- <fsummary>Initializes Config for property testing.</fsummary>
+ <fsummary>Initializes and extends Config for property testing.</fsummary>
<desc><marker id="init_per_suite-1"/>
- <p>Initializes <c>Config</c> for property testing.</p>
+ <p>Initializes and extends <c>Config</c> for property based testing.</p>
<p>This function investigates if support is available for either
- Quickcheck, PropEr, or Triq. The options
- <c>{property_dir,AbsPath}</c> and <c>{property_test_tool,Tool}</c>
- are set in the <c>Config</c> returned.</p>
+ <url href="http://www.quviq.com">QuickCheck</url>,
+ <url href="https://proper-testing.github.io">PropEr</url>
+ or <url href="https://github.com/krestenkrab/triq">Triq</url> and compiles the
+ properties with the first tool found.
+ It is supposed to be called in the <c>init_per_suite/1</c> function
+ in a CommonTest test suite.
+ </p>
+ <p>Which tools to check for, and in which order could be set with the option
+ <c>{prop_tools, list(eqc|proper|triq)}</c>
+ in the CommonTest configuration <c>Config</c>. The default value is
+ <c>[eqc, proper, triq]</c> with <c>eqc</c> being the first one searched for.
+ </p>
+ <p>If no support is found for any tool, this function returns
+ <c>{skip, Explanation}</c>.
+ </p>
+ <p>If support is found, the option <c>{property_test_tool,ToolModule}</c> with
+ the selected tool main module name (<c>eqc</c>, <c>proper</c> or <c>triq</c>)
+ is added to the list <c>Config</c> which then is returned.
+ </p>
+ <p>The property tests are assumed to be in a subdirectory named
+ <c>property_test</c>.
+ All found Erlang files in that directory are compiled with one of the macros
+ <c>'EQC'</c>, <c>'PROPER'</c> or <c>'TRIQ'</c> set, depending on which tool
+ that is first found. This could make parts of the Erlang property tests
+ code to be included or excluded with the macro directives
+ <c>-ifdef(Macro).</c> or <c>-ifndef(Macro).</c>.
+ </p>
+ <p>The file(s) in the <c>property_test</c> subdirectory could, or should,
+ include the ct_property_test include file:
+ </p>
+ <code>
+-include_lib("common_test/include/ct_property_test.hrl").
+ </code>
+ <p>This included file will:
+ </p>
+ <list>
+ <item>Include the correct tool's include file</item>
+ <item>Set the macro <c>'MOD_eqc'</c> to the correct module name for the
+ selected tool. That is, the macro <c>'MOD_eqc'</c> is set to either
+ <c>eqc</c>, <c>proper</c> or <c>triq</c>.
+ </item>
+ </list>
+ </desc>
+ </func>
- <p>The function is intended to be called in function
- <c>init_per_suite</c> in the test suite.</p>
+ <func>
+ <name since="OTP 17.3">quickcheck(Property, Config) -&gt; true | {fail, Reason}</name>
+ <fsummary>Calls quickcheck and returns the result in a form suitable for
+ Common Test.</fsummary>
+ <desc>
+ <p>Calls the selected tool's function for running the <c>Property</c>. It is usually and
+ by historical reasons called quickcheck, and that is why that name is used in
+ this module (<c>ct_property_test</c>).
+ </p>
+ <p>The result is returned in a form suitable for <c>Common Test</c> test suites.
+ </p>
+ <p>This function is intended to be called in test cases in test suites.
+ </p>
+ </desc>
+ </func>
- <p>The property tests are assumed to be in subdirectory
- <c>property_test</c>.</p>
+ <func>
+ <name since="">present_result(Module, Cmds, Triple, Config) -> Result</name>
+ <fsummary>Presents the result of statem property testing</fsummary>
+ <desc>
+ <p>Same as <seealso marker="#present_result/5"><c>present_result(Module, Cmds, Triple, Config, [])</c></seealso>
+ </p>
</desc>
</func>
<func>
- <name since="OTP 17.3">quickcheck(Property, Config) -&gt; true | {fail, Reason}</name>
- <fsummary>Calls quickcheck and returns the result in a form suitable for
- Common Test.</fsummary>
- <desc><marker id="quickcheck-2"/>
- <p>Calls quickcheck and returns the result in a form suitable for
- <c>Common Test</c>.</p>
+ <name since="">present_result(Module, Cmds, Triple, Config, Options) -> Result</name>
+ <fsummary>Presents the result of statem property testing</fsummary>
+ <type>
+ <v>Module = module()</v>
+ <d></d>
+
+ <v>Cmds =</v>
+ <d>the list of commands generated by the property testing tool, for example
+ by proper:commands/1 or by proper:parallel_commands/1
+ </d>
+
+ <v>Triple =</v>
+ <d>the output from for example proper:run_commands/2 or proper:run_parallel_commands/2</d>
+
+ <v>Config =</v>
+ <d>the Common Test <seealso marker="common_test#Module:Testcase/1">Config</seealso> in test cases.</d>
+
+ <v>Options = [present_option()]</v>
+ <v>present_option() = {print_fun, fun(Format,Args)}</v>
+ <v>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;| {spec, StatisticsSpec}</v>
+ <d>The <c>print_fun</c> defines which function to do the actual printout. The default is
+ <seealso marker="ct#log/2">ct:log/2</seealso>.
+ The <c>spec</c> defines what statistics are to be printed<!--, see the
+ <seealso marker="ct_property_test_chapter#spec_present_result">User's Guide</seealso>-->
+ </d>
+
+ <v>Result = boolean()</v>
+ <d>Is <c>false</c> if the test failed and is <c>true</c> if the test passed</d>
+ </type>
+ <desc>
+ <p>Presents the result of <i>stateful (statem) property testing</i> using the aggregate function in
+ PropEr, QuickCheck or other similar property testing tool.
+ </p>
+ <p>It is assumed to be called inside the property called by
+ <seealso marker="#quickcheck/2">quickcheck/2</seealso>:</p>
+ <code>
+...
+RunResult = run_parallel_commands(?MODULE, Cmds),
+ct_property_test:present_result(?MODULE, Cmds, RunResult, Config)
+...
+ </code>
+ <p>See the <seealso marker="ct_property_test_chapter#stateful1">User's Guide</seealso> for
+ an example of the usage and of the default printout.
+ </p>
+ <p>The <c>StatisticsSpec</c> is a list of the tuples:</p>
+ <list>
+ <item><c>{Title::string(), CollectFun::fun/1}</c></item>
+ <item><c>{Title::string(), FrequencyFun::/0, CollectFun::fun/1}</c></item>
+ </list>
+ <p>Each tuple will produce one table in the order of their places in the list.</p>
+ <list>
+ <item><c>Title</c> will be the title of one result table</item>
- <p>This function is intended to be called in the test cases in the
- test suite.</p>
+ <item><c>CollectFun</c> is called with one argument: the <c>Cmds</c>. It should return
+ a list of the values to be counted. The following pre-defined functions exist:
+ <list>
+ <item><c>ct_property_test:cmnd_names/1</c> returns a list of commands (function calls) generated in the <c>Cmnd</c>
+ sequence, without Module, Arguments and other details.</item>
+ <item><c>ct_property_test:num_calls/1</c> returns a list of the length of commands lists</item>
+ <item><c>ct_property_test:sequential_parallel/1</c> returns a list with information about sequential and
+ parallel parts from <c>Tool:parallel_commands/1,2</c></item>
+ </list>
+ </item>
+
+ <item><c>FrequencyFun/0</c> returns a fun/1 which is supposed to take a list of items as input,
+ and return an iolist wich will be printed as the table. Per default, the number of each item is counted
+ and the percentage is printed for each. The list [a,b,a,a,c] could for example return
+ <pre>
+ ["a 60%\n","b 20%\n","c 20%\n"]</pre>
+ which will be printed by the <c>print_fun</c>.
+ The default <c>print_fun</c> will print it as:
+ <pre>
+ a 60%
+ b 20%
+ c 20%</pre>
+ </item>
+ </list>
+ <p>The default <c>StatisticsSpec</c> is:</p>
+ <list>
+ <item>For sequential commands:
+ <code>
+[{"Function calls", fun cmnd_names/1},
+ {"Length of command sequences", fun print_frequency_ranges/0,
+ fun num_calls/1}]
+ </code></item>
+ <item>For parallel commands:
+ <code>
+[{"Distribution sequential/parallel", fun sequential_parallel/1},
+ {"Function calls", fun cmnd_names/1},
+ {"Length of command sequences", fun print_frequency_ranges/0,
+ fun num_calls/1}]
+ </code></item>
+ </list>
</desc>
</func>
+
</funcs>
</erlref>
-
-
diff --git a/lib/common_test/doc/src/ct_property_test_chapter.xml b/lib/common_test/doc/src/ct_property_test_chapter.xml
new file mode 100644
index 0000000000..131f3a962d
--- /dev/null
+++ b/lib/common_test/doc/src/ct_property_test_chapter.xml
@@ -0,0 +1,249 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE chapter SYSTEM "chapter.dtd">
+
+<chapter>
+ <header>
+ <copyright>
+ <year>2019</year><year>2019</year>
+ <holder>Ericsson AB. All Rights Reserved.</holder>
+ </copyright>
+ <legalnotice>
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ </legalnotice>
+
+ <title>Common Test's Property Testing Support: ct_property_test</title>
+ <prepared>Hans Nilsson</prepared>
+ <docno></docno>
+ <date></date>
+ <rev></rev>
+ <file>ct_property_test_chapter.xml</file>
+ </header>
+
+ <section>
+ <marker id="general"></marker>
+ <title>General</title>
+ <p>
+ The <em>Common Test Property Testing Support (ct_property_test)</em>
+ is an aid to run property based testing tools in Common Test test suites.
+ </p>
+ <p>
+ Basic knowledge of property based testing is assumed in the following.
+ It is also assumed that at least one of the following property based
+ testing tools is installed and available in the library path:
+ </p>
+ <list>
+ <item><url href="http://www.quviq.com">QuickCheck</url>,</item>
+ <item><url href="https://proper-testing.github.io">PropEr</url> or</item>
+ <item><url href="https://github.com/krestenkrab/triq">Triq</url></item>
+ </list>
+ </section>
+
+ <section>
+ <marker id="supported"></marker>
+ <title>What Is Supported?</title>
+ <p>The <seealso marker="ct_property_test#">ct_property_test</seealso> module
+ does the following:
+ </p>
+ <list type="bulleted">
+ <item>Compiles the files with property tests in the subdirectory <c>property_test</c>
+ </item>
+ <item>Tests properties in those files using the first found Property Testing Tool.
+ </item>
+ <item>Saves the results - that is the printouts - in the usual Common Test Log
+ </item>
+ </list>
+ </section>
+
+
+ <section>
+ <title>Introductory Example</title>
+ <p>Assume that we want to test the lists:sort/1 function.
+ </p>
+ <p>We need a property to test the function. In normal way, we create
+ <c>property_test/ct_prop.erl</c> module in the <c>test</c> directory
+ in our application:
+ </p>
+
+ <code>
+-module(ct_prop).
+-export([prop_sort/0]).
+
+%%% This will include the .hrl file for the installed testing tool:
+-include_lib("common_test/include/ct_property_test.hrl").
+
+%%% The property we want to check:
+%%% For all possibly unsorted lists,
+%%% the result of lists:sort/1 is sorted.
+prop_sort() -&gt;
+ ?FORALL(UnSorted, list(),
+ is_sorted(lists:sort(UnSorted))
+ ).
+
+%%% Function to check that a list is sorted:
+is_sorted([]) ->
+ true;
+is_sorted([_]) ->
+ true;
+is_sorted([H1,H2|SortedTail]) when H1 =&lt; H2 ->
+ is_sorted([H2|SortedTail]);
+is_sorted(_) ->
+ false.
+ </code>
+
+ <p>We also need a CommonTest test suite:
+ </p>
+ <code>
+-module(ct_property_test_SUITE).
+-compile(export_all). % Only in tests!
+
+-include_lib("common_test/include/ct.hrl").
+
+all() -> [prop_sort
+ ].
+
+%%% First prepare Config and compile the property tests for the found tool:
+init_per_suite(Config) ->
+ ct_property_test:init_per_suite(Config).
+
+end_per_suite(Config) ->
+ Config.
+
+%%%================================================================
+%%% Test suites
+%%%
+prop_sort(Config) ->
+ ct_property_test:quickcheck(
+ ct_prop:prop_sort(),
+ Config
+ ).
+ </code>
+
+ <p>We run it as usual, for example with ct_run in the OS shell:</p>
+ <pre>
+..../test$ ct_run -suite ct_property_test_SUITE
+.....
+Common Test: Running make in test directories...
+
+TEST INFO: 1 test(s), 1 case(s) in 1 suite(s)
+
+Testing lib.common_test.ct_property_test_SUITE: Starting test, 1 test cases
+
+----------------------------------------------------
+2019-12-18 10:44:46.293
+Found property tester proper
+at "/home/X/lib/proper/ebin/proper.beam"
+
+
+----------------------------------------------------
+2019-12-18 10:44:46.294
+Compiling in "/home/..../test/property_test"
+ Deleted: ["ct_prop.beam"]
+ ErlFiles: ["ct_prop.erl"]
+ MacroDefs: [{d,'PROPER'}]
+
+Testing lib.common_test.ct_property_test_SUITE: TEST COMPLETE, 1 ok, 0 failed of 1 test cases
+
+....
+ </pre>
+ </section>
+
+
+ <section>
+ <marker id="stateful1"></marker>
+ <title>A stateful testing example</title>
+ <p>Assume a test that generates some parallel stateful commands, and runs 300 tests:</p>
+ <code>
+prop_parallel(Config) ->
+ numtests(300,
+ ?FORALL(Cmds, parallel_commands(?MODULE),
+ begin
+ RunResult = run_parallel_commands(?MODULE, Cmds),
+ ct_property_test:present_result(?MODULE, Cmds, RunResult, Config)
+ end)).
+ </code>
+ <p>The
+ <seealso marker="ct_property_test#present_result/4">ct_property_test:present_result/4</seealso>
+ is a help function for printing some statistics in the CommonTest log file.</p>
+ <p>Our example test could for example be a simple test of an ftp server, where we perform get, put
+ and delete requests, some of them in parallel. Per default, the result has three sections:
+ </p>
+ <pre>
+*** User 2019-12-11 13:28:17.504 ***
+
+Distribution sequential/parallel
+
+ 57.7% sequential
+ 28.0% parallel_2
+ 14.3% parallel_1
+
+
+
+*** User 2019-12-11 13:28:17.505 ***
+
+Function calls
+
+ 44.4% get
+ 39.3% put
+ 16.3% delete
+
+
+
+*** User 2019-12-11 13:28:17.505 ***
+
+Length of command sequences
+
+Range : Number in range
+-------:----------------
+ 0 - 4: 8 2.7% &lt;-- min=3
+ 5 - 9: 44 14.7%
+10 - 14: 74 24.7%
+15 - 19: 60 20.0% &lt;-- mean=18.7 &lt;-- median=16.0
+20 - 24: 38 12.7%
+25 - 29: 26 8.7%
+30 - 34: 19 6.3%
+35 - 39: 19 6.3%
+40 - 44: 8 2.7%
+45 - 49: 4 1.3% &lt;-- max=47
+ ------
+ 300
+ </pre>
+ <p>The first part - <i>Distribution sequential/parallel</i> - shows the distribution in the
+ sequential and parallel part of the result of parallel_commands/1. See any property testing tool for
+ an explanation of this function.
+ The table shows that of all commands (get and put in our case),
+ 57.7% are executed in the sequential part prior to the parallel part,
+ 28.0% are executed in the first parallel list and the rest in the second parallel list.
+ </p>
+
+ <p>The second part - <i>Function calls</i> - shows the distribution of the three calls in the
+ generated command lists. We see that all of the three calls are executed. If it was so that we
+ thought that we also generated a fourth call, a table like this shows that we failed with that.
+ </p>
+
+ <p>The third and final part - <i>Length of command sequences</i> - show statistics of the
+ generated command sequences. We see that the shortest list has three elementes while the longest
+ has 47 elements. The mean and median values are also shown. Further we could for example see that
+ only 2.7% of the lists (that is eight lists) only has three or four elements.
+ </p>
+
+ </section>
+
+ <!--section>
+ <marker id="spec_present_result"></marker>
+ <title>The spec for present_result/5</title>
+ <p>To be written...
+ <seealso marker="ct_property_test#present_result/5">present_result/5</seealso>
+ </p>
+ </section-->
+</chapter>
diff --git a/lib/common_test/doc/src/part.xml b/lib/common_test/doc/src/part.xml
index 000eb06b82..66dcf75258 100644
--- a/lib/common_test/doc/src/part.xml
+++ b/lib/common_test/doc/src/part.xml
@@ -48,6 +48,7 @@
<xi:include href="dependencies_chapter.xml"/>
<xi:include href="ct_hooks_chapter.xml"/>
<xi:include href="why_test_chapter.xml"/>
+ <xi:include href="ct_property_test_chapter.xml"/>
</part>
diff --git a/lib/common_test/include/ct_property_test.hrl b/lib/common_test/include/ct_property_test.hrl
new file mode 100644
index 0000000000..9d5933fde3
--- /dev/null
+++ b/lib/common_test/include/ct_property_test.hrl
@@ -0,0 +1,40 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2004-2019. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+%%
+
+-ifndef(CT_PROPERTY_TEST_HRL).
+ -define(CT_PROPERTY_TEST_HRL, true).
+
+ -ifdef(EQC).
+ -define(MOD_eqc, eqc).
+ -include_lib("eqc/include/eqc.hrl").
+ -else.
+ -ifdef(PROPER).
+ -define(MOD_eqc, proper).
+ -include_lib("proper/include/proper.hrl").
+ -else.
+ -ifdef(TRIQ).
+ -define(MOD_eqc, triq).
+ -include_lib("triq/include/triq.hrl").
+ -endif.
+ -endif.
+ -endif.
+
+-endif.
diff --git a/lib/common_test/src/Makefile b/lib/common_test/src/Makefile
index 76689dab8c..ffdef8ec39 100644
--- a/lib/common_test/src/Makefile
+++ b/lib/common_test/src/Makefile
@@ -96,7 +96,8 @@ HRL_FILES = \
ct_netconfc.hrl
EXTERNAL_HRL_FILES = \
../include/ct.hrl \
- ../include/ct_event.hrl
+ ../include/ct_event.hrl \
+ ../include/ct_property_test.hrl
EXTERNAL_INC_PATH = ../include
diff --git a/lib/common_test/src/ct_property_test.erl b/lib/common_test/src/ct_property_test.erl
index 1bcae300a8..b8bb890add 100644
--- a/lib/common_test/src/ct_property_test.erl
+++ b/lib/common_test/src/ct_property_test.erl
@@ -18,25 +18,37 @@
%% %CopyrightEnd%
%%
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%%% %%%
-%%% WARNING %%%
-%%% %%%
-%%% This is experimental code which may be changed or removed %%%
-%%% anytime without any warning. %%%
-%%% %%%
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-
-module(ct_property_test).
-%% API
+%%% API
+%% Main functions
-export([init_per_suite/1,
init_tool/1,
quickcheck/2]).
+%% Result presentation
+-export([present_result/4, present_result/5,
+ title/2, title/3,
+ sequential_parallel/1,
+ cmnd_names/1,
+ num_calls/1,
+ print_frequency_ranges/0,
+ print_frequency/0
+ ]).
+
+%%% Mandatory include
-include_lib("common_test/include/ct.hrl").
+%%%================================================================
+%%%
+%%% API
+%%%
+
+%%%----------------------------------------------------------------
+%%%
+%%% Search for a property tester in the lib path, and if found, compile
+%%% the property tests
+%%%
init_per_suite(Config) ->
case init_tool(Config) of
{skip, _}=Skip ->
@@ -55,8 +67,9 @@ init_per_suite(Config) ->
end.
init_tool(Config) ->
- case which_module_exists([eqc,proper,triq]) of
- {ok, ToolModule} ->
+ ToolsToCheck = proplists:get_value(prop_tools, Config, [eqc,proper,triq]),
+ case which_module_exists(ToolsToCheck) of
+ {ok,ToolModule} ->
case code:where_is_file(lists:concat([ToolModule,".beam"])) of
non_existing ->
ct:log("Found ~p, but ~tp~n is not found",
@@ -73,12 +86,71 @@ init_tool(Config) ->
{skip, "No property testing tool found"}
end.
+%%%----------------------------------------------------------------
+%%%
+%%% Call the found property tester (if any)
+%%%
quickcheck(Property, Config) ->
Tool = proplists:get_value(property_test_tool,Config),
F = function_name(quickcheck, Tool),
mk_ct_return( Tool:F(Property), Tool ).
+%%%----------------------------------------------------------------
+%%%
+%%% Present a nice table of the statem result
+%%%
+present_result(Module, Cmds, Triple, Config) ->
+ present_result(Module, Cmds, Triple, Config, []).
+
+present_result(Module, Cmds, {H,Sf,Result}, Config, Options0) ->
+ DefSpec =
+ if
+ is_tuple(Cmds) ->
+ [{"Distribution sequential/parallel", fun sequential_parallel/1}];
+ is_list(Cmds) ->
+ []
+ end
+ ++ [{"Function calls", fun cmnd_names/1},
+ {"Length of command sequences", fun print_frequency_ranges/0, fun num_calls/1}
+ ],
+ Options = add_default_options(Options0,
+ [{print_fun, fun ct:log/2},
+ {spec, DefSpec}
+ ]),
+ do_present_result(Module, Cmds, H, Sf, Result, Config, Options).
+
+
+title(Str, Fun) ->
+ title(Str, Fun, fun io:format/2).
+
+title(Str, Fun, PrintFun) ->
+ fun(L) -> PrintFun("~n~s~n~n~s~n", [Str,Fun(L)]) end.
+
+print_frequency() ->
+ fun(L) ->
+ [io_lib:format("~5.1f% ~p~n",[Pcnt,V])
+ || {V,_Num,Pcnt} <-
+ with_percentage(get_frequencies_no_range(L), length(L))
+ ]
+ end.
+
+print_frequency_ranges() ->
+ print_frequency_ranges([{ngroups,10}]).
+
+print_frequency_ranges(Options0) ->
+ fun([]) ->
+ io_lib:format('Empty list!~n',[]);
+ (L ) ->
+ try
+ Options = set_default_print_freq_range_opts(Options0, L),
+ do_print_frequency_ranges(L, Options)
+ catch
+ C:E:S ->
+ ct:pal("~p:~p ~p:~p~n~p~n~p",[?MODULE,?LINE,C,E,S,L])
+ end
+ end.
+
%%%================================================================
%%%
%%% Local functions
@@ -163,3 +235,217 @@ macro_def(triq) -> [{d, 'TRIQ'}].
function_name(quickcheck, triq) -> check;
function_name(F, _) -> F.
+
+%%%================================================================
+%%%================================================================
+%%%================================================================
+%%%
+%%% Result presentation part
+%%%
+do_present_result(_Module, Cmds, _H, _Sf, ok, Config, Options) ->
+ [PrintFun, Spec] = [proplists:get_value(K,Options) || K <- [print_fun,spec]],
+ Tool = proplists:get_value(property_test_tool,Config),
+ AGGREGATE = function_name(aggregate, Tool),
+ lists:foldr(fun({Title, FreqFun, CollecFun}, Result) ->
+ Tool:AGGREGATE(title(Title, FreqFun(), PrintFun),
+ CollecFun(Cmds),
+ Result);
+ ({Title, CollecFun}, Result) ->
+ Tool:AGGREGATE(title(Title, print_frequency(), PrintFun),
+ CollecFun(Cmds),
+ Result)
+ end, true, Spec);
+
+do_present_result(Module, Cmds, H, Sf, Result, _Config, Options) ->
+ [PrintFun] = [proplists:get_value(K,Options) || K <- [print_fun]],
+ PrintFun("Module = ~p,~n"
+ "Commands = ~p,~n"
+ "History = ~p,~n"
+ "FinalDynState = ~p,~n"
+ "Result = ~p",
+ [Module, Cmds, H, Sf, Result]),
+ Result == ok. % Proper dislikes non-boolean results while eqc treats non-true as false.
+
+%%%================================================================
+cmnd_names(Cs) -> traverse_commands(fun cmnd_name/1, Cs).
+cmnd_name(L) -> [F || {set,_Var,{call,_Mod,F,_As}} <- L].
+
+num_calls(Cs) -> traverse_commands(fun num_call/1, Cs).
+num_call(L) -> [length(L)].
+
+sequential_parallel(Cs) ->
+ traverse_commands(fun(L) -> dup_module(L, sequential) end,
+ fun(L) -> [dup_module(L1, mkmod("parallel",num(L1,L))) || L1<-L] end,
+ Cs).
+dup_module(L, ModName) -> lists:duplicate(length(L), ModName).
+mkmod(PfxStr,N) -> list_to_atom(PfxStr++"_"++integer_to_list(N)).
+
+%% Meta functions for the aggregate functions
+traverse_commands(Fun, L) when is_list(L) -> Fun(L);
+traverse_commands(Fun, {Seq, ParLs}) -> Fun(lists:append([Seq|ParLs])).
+
+traverse_commands(Fseq, _Fpar, L) when is_list(L) -> Fseq(L);
+traverse_commands(Fseq, Fpar, {Seq, ParLs}) -> lists:append([Fseq(Seq)|Fpar(ParLs)]).
+
+%%%================================================================
+-define(middle_dot, 0183).
+
+set_default_print_freq_range_opts(Opts0, L) ->
+ add_default_options(Opts0, [{ngroups, 10},
+ {min, 0},
+ {max, max_in_list(L)}
+ ]).
+
+add_default_options(Opts0, DefaultOpts) ->
+ [set_def_opt(Key,DefVal,Opts0) || {Key,DefVal} <- DefaultOpts].
+
+set_def_opt(Key, DefaultValue, Opts) ->
+ {Key, proplists:get_value(Key, Opts, DefaultValue)}.
+
+max_in_list(L) ->
+ case lists:last(L) of
+ Max when is_integer(Max) -> Max;
+ {Max,_} -> Max
+ end.
+
+do_print_frequency_ranges(L0, Options) ->
+ [N,Min,Max] = [proplists:get_value(K,Options) || K <- [ngroups, min, max]],
+ L = if
+ N>Max ->
+ %% There will be less than the demanded number of classes,
+ %% insert one last with zero values in it. That will force
+ %% the generation of N classes.
+ L0++[{N,0}];
+ N=<Max ->
+ L0
+ end,
+ try
+ Interval = round((Max-Min)/N),
+ IntervalLowerLimits = lists:seq(Min,Max,Interval),
+ Ranges = [{I,I+Interval-1} || I <- IntervalLowerLimits],
+ Acc0 = [{Rng,0} || Rng <- Ranges],
+ Fs0 = get_frequencies(L, Acc0),
+ SumVal = lists:sum([V||{_,V}<-Fs0]),
+ Fs = with_percentage(Fs0, SumVal),
+ DistInfo = [{"min", lists:min(L)},
+ {"mean", mean(L)},
+ {"median", median(L)},
+ {"max", lists:max(L)}],
+
+ Npos_value = num_digits(SumVal),
+ Npos_range = num_digits(Max),
+ [%% Table heading:
+ io_lib:format("Range~*s: ~s~n",[2*Npos_range-2,"", "Number in range"]),
+ %% Line under heading:
+ io_lib:format("~*c:~*c~n",[2*Npos_range+3,$-, max(16,Npos_value+10),$- ]),
+ %% Lines with values:
+ [io_lib:format("~*w - ~*w: ~*w ~5.1f% ~s~n",
+ [Npos_range,Rlow,
+ Npos_range,Rhigh,
+ Npos_value,Val,
+ Percent,
+ cond_prt_vals(DistInfo, Interv)
+ ])
+ || {Interv={Rlow,Rhigh},Val,Percent} <- Fs],
+ %% Line under the table for the total number of values:
+ io_lib:format('~*c ~*c~n',[2*Npos_range,32, Npos_value+3,$-]),
+ %% The total number of values:
+ io_lib:format('~*c ~*w~n',[2*Npos_range,32, Npos_value,SumVal])
+ ]
+ catch
+ C:E ->
+ ct:pal('*** Failed printing (~p:~p) for~n~p~n',[C,E,L])
+ end.
+
+cond_prt_vals(LVs, CurrentInterval) ->
+ [prt_val(Label, Value, CurrentInterval) || {Label,Value} <- LVs].
+
+prt_val(Label, Value, CurrentInterval) ->
+ case in_interval(Value, CurrentInterval) of
+ true ->
+ io_lib:format(" <-- ~s=" ++ if
+ is_float(Value) -> "~.1f";
+ true -> "~p"
+ end,
+ [Label,Value]);
+ false ->
+ ""
+ end.
+
+get_frequencies([{I,Num}|T], [{{Lower,Upper},Cnt}|Acc]) when Lower=<I,I=<Upper ->
+ get_frequencies(T, [{{Lower,Upper},Cnt+Num}|Acc]);
+get_frequencies(L=[{I,_Num}|_], [Ah={{_Lower,Upper},_Cnt}|Acc]) when I>Upper ->
+ [Ah | get_frequencies(L,Acc)];
+get_frequencies([I|T], Acc) when is_integer(I) ->
+ get_frequencies([{I,1}|T], Acc);
+get_frequencies([], Acc) ->
+ Acc.
+
+get_frequencies_no_range([]) ->
+ io_lib:format("No values~n", []);
+get_frequencies_no_range(L) ->
+ [H|T] = lists:sort(L),
+ get_frequencies_no_range(T, H, 1, []).
+
+get_frequencies_no_range([H|T], H, N, Acc) ->
+ get_frequencies_no_range(T, H, N+1, Acc);
+get_frequencies_no_range([H1|T], H, N, Acc) ->
+ get_frequencies_no_range(T, H1, 1, [{H,N}|Acc]);
+get_frequencies_no_range([], H, N, Acc) ->
+ lists:reverse(
+ lists:keysort(2, [{H,N}|Acc])).
+
+%% get_frequencies_percent(L) ->
+%% with_percentage(get_frequencies_no_range(L), length(L)).
+
+
+with_percentage(Fs, Sum) ->
+ [{Rng,Val,100*Val/Sum} || {Rng,Val} <- Fs].
+
+
+num_digits(I) -> 1+trunc(math:log(I)/math:log(10)).
+
+num(Elem, List) -> length(lists:takewhile(fun(E) -> E /= Elem end, List)) + 1.
+
+%%%---- Just for naming an operation for readability
+is_odd(I) -> (I rem 2) == 1.
+
+in_interval(Value, {Rlow,Rhigh}) ->
+ try
+ Rlow=<round(Value) andalso round(Value)=<Rhigh
+ catch
+ _:_ -> false
+ end.
+
+%%%================================================================
+%%% Statistical functions
+
+%%%---- Mean value
+mean(L = [X|_]) when is_number(X) ->
+ lists:sum(L) / length(L);
+mean(L = [{_Value,_Weight}|_]) ->
+ SumOfWeights = lists:sum([W||{_,W}<-L]),
+ WeightedSum = lists:sum([W*V||{V,W}<-L]),
+ WeightedSum / SumOfWeights;
+mean(_) ->
+ undefined.
+
+%%%---- Median
+median(L = [X|_]) when is_number(X) ->
+ Len = length(L),
+ case is_odd(Len) of
+ true ->
+ hd(lists:nthtail(Len div 2, L));
+ false ->
+ %% 1) L has at least one element (the one in the is_number test).
+ %% 2) Length is even.
+ %% => Length >= 2
+ [M1,M2|_] = lists:nthtail((Len div 2)-1, L),
+ (M1+M2) / 2
+ end;
+%% integer Weights...
+median(L = [{_Value,_Weight}|_]) ->
+ median( lists:append([lists:duplicate(W,V) || {V,W} <- L]) );
+median(_) ->
+ undefined.
+
diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile
index e510b74d6a..fae7ce0eb5 100644
--- a/lib/common_test/test/Makefile
+++ b/lib/common_test/test/Makefile
@@ -76,7 +76,8 @@ MODULES= \
ct_unicode_SUITE \
ct_auto_clean_SUITE \
ct_util_SUITE \
- ct_tc_repeat_SUITE
+ ct_tc_repeat_SUITE \
+ ct_property_test_SUITE
ERL_FILES= $(MODULES:%=%.erl)
HRL_FILES= test_server_test_lib.hrl
diff --git a/lib/common_test/test/ct_property_test_SUITE.erl b/lib/common_test/test/ct_property_test_SUITE.erl
new file mode 100644
index 0000000000..1f8c9e08cf
--- /dev/null
+++ b/lib/common_test/test/ct_property_test_SUITE.erl
@@ -0,0 +1,24 @@
+-module(ct_property_test_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+all() -> [prop_sort
+ ].
+
+%%% First prepare Config and compile the property tests for the found tool:
+init_per_suite(Config) ->
+ ct_property_test:init_per_suite(Config).
+
+end_per_suite(Config) ->
+ Config.
+
+%%%================================================================
+%%% Test suites
+%%%
+prop_sort(Config) ->
+ ct_property_test:quickcheck(
+ ct_prop:prop_sort(),
+ Config
+ ).
diff --git a/lib/common_test/test/property_test/ct_prop.erl b/lib/common_test/test/property_test/ct_prop.erl
new file mode 100644
index 0000000000..67ab3f3e6b
--- /dev/null
+++ b/lib/common_test/test/property_test/ct_prop.erl
@@ -0,0 +1,18 @@
+-module(ct_prop).
+-export([prop_sort/0]).
+
+-include_lib("common_test/include/ct_property_test.hrl").
+
+prop_sort() ->
+ ?FORALL(UnSorted, list(),
+ is_sorted(lists:sort(UnSorted))
+ ).
+
+is_sorted([]) ->
+ true;
+is_sorted([_]) ->
+ true;
+is_sorted([H1,H2|SortedTail]) when H1 =< H2 ->
+ is_sorted([H2|SortedTail]);
+is_sorted(_) ->
+ false.