From 6f49a634a7ce310624748b90299d23ccdbf7f743 Mon Sep 17 00:00:00 2001 From: Hans Nilsson Date: Wed, 11 Dec 2019 13:00:04 +0100 Subject: common_test: Presentation function --- lib/common_test/src/ct_property_test.erl | 300 ++++++++++++++++++++++++++++++- 1 file changed, 298 insertions(+), 2 deletions(-) diff --git a/lib/common_test/src/ct_property_test.erl b/lib/common_test/src/ct_property_test.erl index 93642a0970..5acf380367 100644 --- a/lib/common_test/src/ct_property_test.erl +++ b/lib/common_test/src/ct_property_test.erl @@ -30,12 +30,35 @@ -module(ct_property_test). -%% API +%%% API +%% Main functions -export([init_per_suite/1, - quickcheck/2]). + quickcheck/2 + ]). +%% Result presentation +-export([present_result/4, present_result/5, + title/2, title/3, + sequential_parallel/1, + cmnd_names/1, + num_calls/1, + print_frequency_ranges/0, + print_frequency/0 + ]). + +%%% Mandatory include -include_lib("common_test/include/ct.hrl"). +%%%================================================================ +%%% +%%% API +%%% + +%%%---------------------------------------------------------------- +%%% +%%% Search for a property tester in the lib path, and if found, compile +%%% the property tests +%%% init_per_suite(Config) -> case which_module_exists([eqc,proper,triq]) of {ok,ToolModule} -> @@ -66,12 +89,71 @@ init_per_suite(Config) -> {skip, "No property testing tool found"} end. +%%%---------------------------------------------------------------- +%%% +%%% Call the found property tester (if any) +%%% quickcheck(Property, Config) -> Tool = proplists:get_value(property_test_tool,Config), F = function_name(quickcheck, Tool), mk_ct_return( Tool:F(Property), Tool ). +%%%---------------------------------------------------------------- +%%% +%%% Present a nice table of the statem result +%%% +present_result(Module, Cmds, Triple, Config) -> + present_result(Module, Cmds, Triple, Config, []). + +present_result(Module, Cmds, {H,Sf,Result}, Config, Options0) -> + DefSpec = + if + is_tuple(Cmds) -> + [{"Distribution sequential/parallel", fun sequential_parallel/1}]; + is_list(Cmds) -> + [] + end + ++ [{"Function calls", fun cmnd_names/1}, + {"Length of command sequences", fun print_frequency_ranges/0, fun num_calls/1} + ], + Options = add_default_options(Options0, + [{print_fun, fun ct:log/2}, + {spec, DefSpec} + ]), + do_present_result(Module, Cmds, H, Sf, Result, Config, Options). + + +title(Str, Fun) -> + title(Str, Fun, fun io:format/2). + +title(Str, Fun, PrintFun) -> + fun(L) -> PrintFun("~n~s~n~n~s~n", [Str,Fun(L)]) end. + +print_frequency() -> + fun(L) -> + [io_lib:format("~5.1f% ~p~n",[Pcnt,V]) + || {V,_Num,Pcnt} <- + with_percentage(get_frequencies_no_range(L), length(L)) + ] + end. + +print_frequency_ranges() -> + print_frequency_ranges([{ngroups,10}]). + +print_frequency_ranges(Options0) -> + fun([]) -> + io_lib:format('Empty list!~n',[]); + (L ) -> + try + Options = set_default_print_freq_range_opts(Options0, L), + do_print_frequency_ranges(L, Options) + catch + C:E:S -> + ct:pal("~p:~p ~p:~p~n~p~n~p",[?MODULE,?LINE,C,E,S,L]) + end + end. + %%%================================================================ %%% %%% Local functions @@ -155,3 +237,217 @@ macro_def(triq) -> [{d, 'TRIQ'}]. function_name(quickcheck, triq) -> check; function_name(F, _) -> F. + +%%%================================================================ +%%%================================================================ +%%%================================================================ +%%% +%%% Result presentation part +%%% +do_present_result(_Module, Cmds, _H, _Sf, ok, Config, Options) -> + [PrintFun, Spec] = [proplists:get_value(K,Options) || K <- [print_fun,spec]], + Tool = proplists:get_value(property_test_tool,Config), + AGGREGATE = function_name(aggregate, Tool), + lists:foldr(fun({Title, FreqFun, CollecFun}, Result) -> + Tool:AGGREGATE(title(Title, FreqFun(), PrintFun), + CollecFun(Cmds), + Result); + ({Title, CollecFun}, Result) -> + Tool:AGGREGATE(title(Title, print_frequency(), PrintFun), + CollecFun(Cmds), + Result) + end, true, Spec); + +do_present_result(Module, Cmds, H, Sf, Result, _Config, Options) -> + [PrintFun] = [proplists:get_value(K,Options) || K <- [print_fun]], + PrintFun("Module = ~p,~n" + "Commands = ~p,~n" + "History = ~p,~n" + "FinalDynState = ~p,~n" + "Result = ~p", + [Module, Cmds, H, Sf, Result]), + Result == ok. % Proper dislikes non-boolean results while eqc treats non-true as false. + +%%%================================================================ +cmnd_names(Cs) -> traverse_commands(fun cmnd_name/1, Cs). +cmnd_name(L) -> [F || {set,_Var,{call,_Mod,F,_As}} <- L]. + +num_calls(Cs) -> traverse_commands(fun num_call/1, Cs). +num_call(L) -> [length(L)]. + +sequential_parallel(Cs) -> + traverse_commands(fun(L) -> dup_module(L, sequential) end, + fun(L) -> [dup_module(L1, mkmod("parallel",num(L1,L))) || L1<-L] end, + Cs). +dup_module(L, ModName) -> lists:duplicate(length(L), ModName). +mkmod(PfxStr,N) -> list_to_atom(PfxStr++"_"++integer_to_list(N)). + +%% Meta functions for the aggregate functions +traverse_commands(Fun, L) when is_list(L) -> Fun(L); +traverse_commands(Fun, {Seq, ParLs}) -> Fun(lists:append([Seq|ParLs])). + +traverse_commands(Fseq, _Fpar, L) when is_list(L) -> Fseq(L); +traverse_commands(Fseq, Fpar, {Seq, ParLs}) -> lists:append([Fseq(Seq)|Fpar(ParLs)]). + +%%%================================================================ +-define(middle_dot, 0183). + +set_default_print_freq_range_opts(Opts0, L) -> + add_default_options(Opts0, [{ngroups, 10}, + {min, 0}, + {max, max_in_list(L)} + ]). + +add_default_options(Opts0, DefaultOpts) -> + [set_def_opt(Key,DefVal,Opts0) || {Key,DefVal} <- DefaultOpts]. + +set_def_opt(Key, DefaultValue, Opts) -> + {Key, proplists:get_value(Key, Opts, DefaultValue)}. + +max_in_list(L) -> + case lists:last(L) of + Max when is_integer(Max) -> Max; + {Max,_} -> Max + end. + +do_print_frequency_ranges(L0, Options) -> + [N,Min,Max] = [proplists:get_value(K,Options) || K <- [ngroups, min, max]], + L = if + N>Max -> + %% There will be less than the demanded number of classes, + %% insert one last with zero values in it. That will force + %% the generation of N classes. + L0++[{N,0}]; + N= + L0 + end, + try + Interval = round((Max-Min)/N), + IntervalLowerLimits = lists:seq(Min,Max,Interval), + Ranges = [{I,I+Interval-1} || I <- IntervalLowerLimits], + Acc0 = [{Rng,0} || Rng <- Ranges], + Fs0 = get_frequencies(L, Acc0), + SumVal = lists:sum([V||{_,V}<-Fs0]), + Fs = with_percentage(Fs0, SumVal), + DistInfo = [{"min", lists:min(L)}, + {"mean", mean(L)}, + {"median", median(L)}, + {"max", lists:max(L)}], + + Npos_value = num_digits(SumVal), + Npos_range = num_digits(Max), + [%% Table heading: + io_lib:format("Range~*s: ~s~n",[2*Npos_range-2,"", "Number in range"]), + %% Line under heading: + io_lib:format("~*c:~*c~n",[2*Npos_range+3,$-, max(16,Npos_value+10),$- ]), + %% Lines with values: + [io_lib:format("~*w - ~*w: ~*w ~5.1f% ~s~n", + [Npos_range,Rlow, + Npos_range,Rhigh, + Npos_value,Val, + Percent, + cond_prt_vals(DistInfo, Interv) + ]) + || {Interv={Rlow,Rhigh},Val,Percent} <- Fs], + %% Line under the table for the total number of values: + io_lib:format('~*c ~*c~n',[2*Npos_range,32, Npos_value+3,$-]), + %% The total number of values: + io_lib:format('~*c ~*w~n',[2*Npos_range,32, Npos_value,SumVal]) + ] + catch + C:E -> + ct:pal('*** Failed printing (~p:~p) for~n~p~n',[C,E,L]) + end. + +cond_prt_vals(LVs, CurrentInterval) -> + [prt_val(Label, Value, CurrentInterval) || {Label,Value} <- LVs]. + +prt_val(Label, Value, CurrentInterval) -> + case in_interval(Value, CurrentInterval) of + true -> + io_lib:format(" <-- ~s=" ++ if + is_float(Value) -> "~.1f"; + true -> "~p" + end, + [Label,Value]); + false -> + "" + end. + +get_frequencies([{I,Num}|T], [{{Lower,Upper},Cnt}|Acc]) when Lower= + get_frequencies(T, [{{Lower,Upper},Cnt+Num}|Acc]); +get_frequencies(L=[{I,_Num}|_], [Ah={{_Lower,Upper},_Cnt}|Acc]) when I>Upper -> + [Ah | get_frequencies(L,Acc)]; +get_frequencies([I|T], Acc) when is_integer(I) -> + get_frequencies([{I,1}|T], Acc); +get_frequencies([], Acc) -> + Acc. + +get_frequencies_no_range([]) -> + io_lib:format("No values~n", []); +get_frequencies_no_range(L) -> + [H|T] = lists:sort(L), + get_frequencies_no_range(T, H, 1, []). + +get_frequencies_no_range([H|T], H, N, Acc) -> + get_frequencies_no_range(T, H, N+1, Acc); +get_frequencies_no_range([H1|T], H, N, Acc) -> + get_frequencies_no_range(T, H1, 1, [{H,N}|Acc]); +get_frequencies_no_range([], H, N, Acc) -> + lists:reverse( + lists:keysort(2, [{H,N}|Acc])). + +%% get_frequencies_percent(L) -> +%% with_percentage(get_frequencies_no_range(L), length(L)). + + +with_percentage(Fs, Sum) -> + [{Rng,Val,100*Val/Sum} || {Rng,Val} <- Fs]. + + +num_digits(I) -> 1+trunc(math:log(I)/math:log(10)). + +num(Elem, List) -> length(lists:takewhile(fun(E) -> E /= Elem end, List)) + 1. + +%%%---- Just for naming an operation for readability +is_odd(I) -> (I rem 2) == 1. + +in_interval(Value, {Rlow,Rhigh}) -> + try + Rlow= false + end. + +%%%================================================================ +%%% Statistical functions + +%%%---- Mean value +mean(L = [X|_]) when is_number(X) -> + lists:sum(L) / length(L); +mean(L = [{_Value,_Weight}|_]) -> + SumOfWeights = lists:sum([W||{_,W}<-L]), + WeightedSum = lists:sum([W*V||{V,W}<-L]), + WeightedSum / SumOfWeights; +mean(_) -> + undefined. + +%%%---- Median +median(L = [X|_]) when is_number(X) -> + Len = length(L), + case is_odd(Len) of + true -> + hd(lists:nthtail(Len div 2, L)); + false -> + %% 1) L has at least one element (the one in the is_number test). + %% 2) Length is even. + %% => Length >= 2 + [M1,M2|_] = lists:nthtail((Len div 2)-1, L), + (M1+M2) / 2 + end; +%% integer Weights... +median(L = [{_Value,_Weight}|_]) -> + median( lists:append([lists:duplicate(W,V) || {V,W} <- L]) ); +median(_) -> + undefined. + -- cgit v1.2.1 From 855912f5056fc2963e6921bd5360481901c2f551 Mon Sep 17 00:00:00 2001 From: Hans Nilsson Date: Mon, 16 Dec 2019 11:55:01 +0100 Subject: common_test: Remove warning for experimental code --- lib/common_test/doc/src/ct_property_test.xml | 12 ++---------- lib/common_test/src/ct_property_test.erl | 10 ---------- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/lib/common_test/doc/src/ct_property_test.xml b/lib/common_test/doc/src/ct_property_test.xml index 1e01d9a5d7..0fd641a6f3 100644 --- a/lib/common_test/doc/src/ct_property_test.xml +++ b/lib/common_test/doc/src/ct_property_test.xml @@ -33,15 +33,12 @@ ct_property_test.xml ct_property_test - EXPERIMENTAL support in Common Test for calling + Support in Common Test for calling property-based tests. -

EXPERIMENTAL support in Common Test for calling property-based - tests.

- -

This module is a first step to run property-based tests in the +

This module helps running property-based tests in the Common Test framework. A property testing tool like QuickCheck or PropEr is assumed to be installed.

@@ -70,11 +67,6 @@ Config ). - -

This is experimental code that can be changed or removed anytime - without any warning.

- - diff --git a/lib/common_test/src/ct_property_test.erl b/lib/common_test/src/ct_property_test.erl index 5acf380367..47e6be30f6 100644 --- a/lib/common_test/src/ct_property_test.erl +++ b/lib/common_test/src/ct_property_test.erl @@ -18,16 +18,6 @@ %% %CopyrightEnd% %% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%%% %%% -%%% WARNING %%% -%%% %%% -%%% This is experimental code which may be changed or removed %%% -%%% anytime without any warning. %%% -%%% %%% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - -module(ct_property_test). %%% API -- cgit v1.2.1 From fe7c44eb8a92bb30ba7ca5b1e23bfaaf4acff221 Mon Sep 17 00:00:00 2001 From: Hans Nilsson Date: Mon, 16 Dec 2019 14:55:24 +0100 Subject: common_test: Add .hrl for support macros and -include of proptest tool --- lib/common_test/include/ct_property_test.hrl | 40 ++++++++++++++++++++++++++++ lib/common_test/src/Makefile | 3 ++- 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 lib/common_test/include/ct_property_test.hrl diff --git a/lib/common_test/include/ct_property_test.hrl b/lib/common_test/include/ct_property_test.hrl new file mode 100644 index 0000000000..9d5933fde3 --- /dev/null +++ b/lib/common_test/include/ct_property_test.hrl @@ -0,0 +1,40 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2004-2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +%% + +-ifndef(CT_PROPERTY_TEST_HRL). + -define(CT_PROPERTY_TEST_HRL, true). + + -ifdef(EQC). + -define(MOD_eqc, eqc). + -include_lib("eqc/include/eqc.hrl"). + -else. + -ifdef(PROPER). + -define(MOD_eqc, proper). + -include_lib("proper/include/proper.hrl"). + -else. + -ifdef(TRIQ). + -define(MOD_eqc, triq). + -include_lib("triq/include/triq.hrl"). + -endif. + -endif. + -endif. + +-endif. diff --git a/lib/common_test/src/Makefile b/lib/common_test/src/Makefile index 76689dab8c..ffdef8ec39 100644 --- a/lib/common_test/src/Makefile +++ b/lib/common_test/src/Makefile @@ -96,7 +96,8 @@ HRL_FILES = \ ct_netconfc.hrl EXTERNAL_HRL_FILES = \ ../include/ct.hrl \ - ../include/ct_event.hrl + ../include/ct_event.hrl \ + ../include/ct_property_test.hrl EXTERNAL_INC_PATH = ../include -- cgit v1.2.1 From eb52758f7fc76dd967f261d66c55c81c10a612de Mon Sep 17 00:00:00 2001 From: Hans Nilsson Date: Mon, 16 Dec 2019 16:41:28 +0100 Subject: common_test: ct_property_test test case --- lib/common_test/test/Makefile | 3 ++- lib/common_test/test/ct_property_test_SUITE.erl | 24 +++++++++++++++++++++ lib/common_test/test/property_test/ct_prop.erl | 28 +++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 lib/common_test/test/ct_property_test_SUITE.erl create mode 100644 lib/common_test/test/property_test/ct_prop.erl diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile index e510b74d6a..fae7ce0eb5 100644 --- a/lib/common_test/test/Makefile +++ b/lib/common_test/test/Makefile @@ -76,7 +76,8 @@ MODULES= \ ct_unicode_SUITE \ ct_auto_clean_SUITE \ ct_util_SUITE \ - ct_tc_repeat_SUITE + ct_tc_repeat_SUITE \ + ct_property_test_SUITE ERL_FILES= $(MODULES:%=%.erl) HRL_FILES= test_server_test_lib.hrl diff --git a/lib/common_test/test/ct_property_test_SUITE.erl b/lib/common_test/test/ct_property_test_SUITE.erl new file mode 100644 index 0000000000..1f8c9e08cf --- /dev/null +++ b/lib/common_test/test/ct_property_test_SUITE.erl @@ -0,0 +1,24 @@ +-module(ct_property_test_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +all() -> [prop_sort + ]. + +%%% First prepare Config and compile the property tests for the found tool: +init_per_suite(Config) -> + ct_property_test:init_per_suite(Config). + +end_per_suite(Config) -> + Config. + +%%%================================================================ +%%% Test suites +%%% +prop_sort(Config) -> + ct_property_test:quickcheck( + ct_prop:prop_sort(), + Config + ). diff --git a/lib/common_test/test/property_test/ct_prop.erl b/lib/common_test/test/property_test/ct_prop.erl new file mode 100644 index 0000000000..559c33da74 --- /dev/null +++ b/lib/common_test/test/property_test/ct_prop.erl @@ -0,0 +1,28 @@ +-module(ct_prop). +-export([prop_sort/0]). + +-include_lib("common_test/include/ct_property_test.hrl"). + +prop_sort() -> + ?FORALL(UnSorted, list(), + is_sorted(lists:sort(UnSorted)) + ). + + +is_sorted([]) -> + true; +is_sorted([_]) -> + true; +is_sorted(Sorted) -> + try + lists:foldl(fun chk_sorted_pair/2, hd(Sorted), tl(Sorted)) + of + _ -> + true + catch + throw:false -> + false + end. + +chk_sorted_pair(A, B) when A>=B -> A; +chk_sorted_pair(_, _) -> throw(false). -- cgit v1.2.1 From 610e8bb1d9bcf7f31e2839a178e5ef3df0b541cb Mon Sep 17 00:00:00 2001 From: Hans Nilsson Date: Tue, 17 Dec 2019 15:47:10 +0100 Subject: common_test: Add 'prop_tools' option to Config for ct_property_test --- lib/common_test/src/ct_property_test.erl | 3 ++- lib/common_test/test/property_test/ct_prop.erl | 18 ++++-------------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/lib/common_test/src/ct_property_test.erl b/lib/common_test/src/ct_property_test.erl index 47e6be30f6..251a0a4896 100644 --- a/lib/common_test/src/ct_property_test.erl +++ b/lib/common_test/src/ct_property_test.erl @@ -50,7 +50,8 @@ %%% the property tests %%% init_per_suite(Config) -> - case which_module_exists([eqc,proper,triq]) of + ToolsToCheck = proplists:get_value(prop_tools, Config, [eqc,proper,triq]), + case which_module_exists(ToolsToCheck) of {ok,ToolModule} -> case code:where_is_file(lists:concat([ToolModule,".beam"])) of non_existing -> diff --git a/lib/common_test/test/property_test/ct_prop.erl b/lib/common_test/test/property_test/ct_prop.erl index 559c33da74..67ab3f3e6b 100644 --- a/lib/common_test/test/property_test/ct_prop.erl +++ b/lib/common_test/test/property_test/ct_prop.erl @@ -8,21 +8,11 @@ prop_sort() -> is_sorted(lists:sort(UnSorted)) ). - is_sorted([]) -> true; is_sorted([_]) -> true; -is_sorted(Sorted) -> - try - lists:foldl(fun chk_sorted_pair/2, hd(Sorted), tl(Sorted)) - of - _ -> - true - catch - throw:false -> - false - end. - -chk_sorted_pair(A, B) when A>=B -> A; -chk_sorted_pair(_, _) -> throw(false). +is_sorted([H1,H2|SortedTail]) when H1 =< H2 -> + is_sorted([H2|SortedTail]); +is_sorted(_) -> + false. -- cgit v1.2.1 From d697201421023112c708b5710d4b2818af27bb57 Mon Sep 17 00:00:00 2001 From: Hans Nilsson Date: Wed, 11 Dec 2019 15:13:21 +0100 Subject: common_test: Document property test reporting --- lib/common_test/doc/src/Makefile | 1 + lib/common_test/doc/src/ct_property_test.xml | 229 ++++++++++++++++--- .../doc/src/ct_property_test_chapter.xml | 249 +++++++++++++++++++++ lib/common_test/doc/src/part.xml | 1 + 4 files changed, 446 insertions(+), 34 deletions(-) create mode 100644 lib/common_test/doc/src/ct_property_test_chapter.xml diff --git a/lib/common_test/doc/src/Makefile b/lib/common_test/doc/src/Makefile index b5acdc6f95..ae06572752 100644 --- a/lib/common_test/doc/src/Makefile +++ b/lib/common_test/doc/src/Makefile @@ -74,6 +74,7 @@ XML_CHAPTER_FILES = \ ct_master_chapter.xml \ event_handler_chapter.xml \ ct_hooks_chapter.xml \ + ct_property_test_chapter.xml \ dependencies_chapter.xml \ notes.xml diff --git a/lib/common_test/doc/src/ct_property_test.xml b/lib/common_test/doc/src/ct_property_test.xml index 0fd641a6f3..687da15b0a 100644 --- a/lib/common_test/doc/src/ct_property_test.xml +++ b/lib/common_test/doc/src/ct_property_test.xml @@ -33,27 +33,39 @@ ct_property_test.xml ct_property_test - Support in Common Test for calling - property-based tests. + Support in Common Test for running property-based tests.

This module helps running property-based tests in the - Common Test framework. A property testing tool like QuickCheck - or PropEr is assumed to be installed.

+ Common Test framework. One (or more) of the property testing tools +

+ + QuickCheck, + PropEr or + Triq + +

+ is assumed to be installed. +

-

The idea is to have a Common Test test suite calling a property - testing tool with special property test suites as defined by that tool. - The usual Erlang application directory structure is assumed. The tests - are collected in the test directory of the application. The - test directory has a subdirectory property_test, where - everything needed for the property tests is collected.

+

The idea with this module is to have a Common Test test suite calling + a property testing tool with special property test suites as defined by that tool. + The tests + are collected in the test directory of the application. The + test directory has a subdirectory property_test, where + everything needed for the property tests are collected. + The usual Erlang application directory structure is assumed. +

A typical Common Test test suite using ct_property_test - is organized as follows:

+ is organized as follows:

- 
- -include_lib("common_test/include/ct.hrl").
+    
+-module(my_prop_test_SUITE).
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
 
  all() -> [prop_ftp_case].
 
@@ -63,46 +75,195 @@
  %%%---- test case
  prop_ftp_case(Config) ->
      ct_property_test:quickcheck(
-       ftp_simple_client_server:prop_ftp(Config),
+       ftp_simple_client_server:prop_ftp(),
        Config
-      ).
+ ). +

and the the property test module (in this example ftp_simple_client_server.erl) + as almost a usual property testing module + (More examples are in the User's Guide):

+ +-module(ftp_simple_client_server). +-export([prop_ftp/0...]). + +-include_lib("common_test/include/ct_property_test.hrl"). +prop_ftp() -> + ?FORALL( .... + init_per_suite(Config) -> Config | {skip, Reason} - Initializes Config for property testing. + Initializes and extends Config for property testing. -

Initializes Config for property testing.

+

Initializes and extends Config for property based testing.

This function investigates if support is available for either - Quickcheck, PropEr, or Triq. The options - {property_dir,AbsPath} and {property_test_tool,Tool} - are set in the Config returned.

+ QuickCheck, + PropEr + or Triq and compiles the + properties with the first tool found. + It is supposed to be called in the init_per_suite/1 function + in a CommonTest test suite. +

+

Which tools to check for, and in which order could be set with the option + {prop_tools, list(eqc|proper|triq)} + in the CommonTest configuration Config. The default value is + [eqc, proper, triq] with eqc being the first one searched for. +

+

If no support is found for any tool, this function returns + {skip, Explanation}. +

+

If support is found, the option {property_test_tool,ToolModule} with + the selected tool main module name (eqc, proper or triq) + is added to the list Config which then is returned. +

+

The property tests are assumed to be in a subdirectory named + property_test. + All found Erlang files in that directory are compiled with one of the macros + 'EQC', 'PROPER' or 'TRIQ' set, depending on which tool + that is first found. +

+

The file(s) in the property_test subdirectory could, or should, + include the ct_property_test include file: +

+ +-include_lib("common_test/include/ct_property_test.hrl"). + +

This included file will: +

+ + Include the correct tool's include file + Set the macro 'MOD_eqc' to the correct module name for the + selected tool, that is eqc, proper or triq. + + + + -

The function is intended to be called in function - init_per_suite in the test suite.

+ + quickcheck(Property, Config) -> true | {fail, Reason} + Calls quickcheck and returns the result in a form suitable for + Common Test. + +

Calls the selected tool's function for running the Property. It is usually and + by historical reasons called quickcheck, and that is why that name is used in + this module (ct_property_test). +

+

The result is returned in a form suitable for Common Test test suites. +

+

This function is intended to be called in test cases in test suites. +

+ + -

The property tests are assumed to be in subdirectory - property_test.

+ + present_result(Module, Cmds, Triple, Config) -> Result + Presents the result of statem property testing + +

Same as present_result(Module, Cmds, Triple, Config, []) +

- quickcheck(Property, Config) -> true | {fail, Reason} - Calls quickcheck and returns the result in a form suitable for - Common Test. - -

Calls quickcheck and returns the result in a form suitable for - Common Test.

+ present_result(Module, Cmds, Triple, Config, Options) -> Result + Presents the result of statem property testing + + Module = module() + + + Cmds = + the list of commands generated by the property testing tool, for example + by proper:commands/1 or by proper:parallel_commands/1 + + + Triple = + the output from for example proper:run_commands/2 or proper:run_parallel_commands/2 + + Config = + the Common Test Config in test cases. + + Options = [present_option()] + present_option() = {print_fun, fun(Format,Args)} +                  | {spec, StatisticsSpec} + The print_fun defines which function to do the actual printout. The default is + ct:log/2. + The spec defines what statistics are to be printed + + + Result = boolean() + Is false if the test failed and is true if the test passed + + +

Presents the result of stateful (statem) property testing using the aggregate function in + PropEr, QuickCheck or other similar property testing tool. +

+

It is assumed to be called inside the property called by + quickcheck/2:

+ +... +RunResult = run_parallel_commands(?MODULE, Cmds), +ct_property_test:present_result(?MODULE, Cmds, RunResult, Config) +... + +

See the User's Guide for + an example of the usage and of the default printout. +

+

The StatisticsSpec is a list of the tuples:

+ + {Title::string(), CollectFun::fun/1} + {Title::string(), FrequencyFun::/0, CollectFun::fun/1} + +

Each tuple will produce one table in the order of their places in the list.

+ + Title will be the title of one result table -

This function is intended to be called in the test cases in the - test suite.

+ CollectFun is called with one argument: the Cmds. It should return + a list of the values to be counted. The following pre-defined functions exist: + + ct_property_test:cmnd_names/1 returns a list of commands (function calls) generated in the Cmnd + sequence, without Module, Arguments and other details. + ct_property_test:num_calls/1 returns a list of the length of commands lists + ct_property_test:sequential_parallel/1 returns a list with information about sequential and + parallel parts from Tool:parallel_commands/1,2 + + + + FrequencyFun/0 returns a fun/1 which is supposed to take a list of items as input, + and return an iolist wich will be printed as the table. Per default, the number of each item is counted + and the percentage is printed for each. The list [a,b,a,a,c] could for example return + 
+ ["a 60%\n","b 20%\n","c 20%\n"]
+ which will be printed by the print_fun. + The default print_fun will print it as: + 
+ a 60%
+ b 20%
+ c 20%
+ + +

The default StatisticsSpec is:

+ + For sequential commands: + +[{"Function calls", fun cmnd_names/1}, + {"Length of command sequences", fun print_frequency_ranges/0, + fun num_calls/1}] + + For parallel commands: + +[{"Distribution sequential/parallel", fun sequential_parallel/1}, + {"Function calls", fun cmnd_names/1}, + {"Length of command sequences", fun print_frequency_ranges/0, + fun num_calls/1}] + + + - - diff --git a/lib/common_test/doc/src/ct_property_test_chapter.xml b/lib/common_test/doc/src/ct_property_test_chapter.xml new file mode 100644 index 0000000000..8cc273d2fc --- /dev/null +++ b/lib/common_test/doc/src/ct_property_test_chapter.xml @@ -0,0 +1,249 @@ + + + + +
+ + 20192019 + Ericsson AB. All Rights Reserved. + + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + + Common Test's Property Testing Support: ct_property_test + Hans Nilsson + + + + ct_property_test_chapter.xml +
+ +
+ + General +

+ The Common Test Property Testing Support (ct_property_test) + is an aid to run property based testing tools in Common Test test suites. +

+

+ Basic knowledge of property based testing is assumed in the following. + It is also assumed that at least one of the following property based + testing tools is installed and available in the library path: +

+ + QuickCheck, + PropEr or + Triq + +
+ +
+ + What Is Supported? +

The ct_property_test module + does the following: +

+ + Compiles the files with property tests in the subdirectory property_test + + Tests properties in those files using the first found Property Testing Tool. + + Saves the results - that is the printouts - in the usual Common Test Log + + +
+ + +
+ Introductory Example +

Assume that we want to test the lists:sort/1 function. +

+

We need a property to test the function. In normal way, we create + property_test/test_sort.erl module in the test directory + in our application: +

+ + +-module(test_sort). +-export([prop_sort/0]). + +%%% This will include the .hrl file for the installed testing tool: +-include_lib("common_test/include/ct_property_test.hrl"). + +%%% The property we want to check: +%%% For all possibly unsorted lists, +%%% the result of lists:sort/1 is sorted. +prop_sort() -> + ?FORALL(UnSorted, list(), + is_sorted(lists:sort(UnSorted)) + ). + +%%% Function to check that a list is sorted: +is_sorted([]) -> + true; +is_sorted([_]) -> + true; +is_sorted([H1,H2|SortedTail]) when H1 =< H2 -> + is_sorted([H2|SortedTail]); +is_sorted(_) -> + false. + + +

We also need a CommonTest test suite: +

+ +-module(ct_property_test_SUITE). +-compile(export_all). % Only in tests! + +-include_lib("common_test/include/ct.hrl"). + +all() -> [prop_sort + ]. + +%%% First prepare Config and compile the property tests for the found tool: +init_per_suite(Config) -> + ct_property_test:init_per_suite(Config). + +end_per_suite(Config) -> + Config. + +%%%================================================================ +%%% Test suites +%%% +prop_sort(Config) -> + ct_property_test:quickcheck( + ct_prop:prop_sort(), + Config + ). + + +

We run it as usual, for example with ct_run in the OS shell:

+ 
+..../test$ ct_run -suite ct_property_test_SUITE
+.....
+Common Test: Running make in test directories...
+
+TEST INFO: 1 test(s), 1 case(s) in 1 suite(s)
+
+Testing lib.common_test.ct_property_test_SUITE: Starting test, 1 test cases
+
+----------------------------------------------------
+2019-12-18 10:44:46.293
+Found property tester proper
+at "/home/X/lib/proper/ebin/proper.beam"
+
+
+----------------------------------------------------
+2019-12-18 10:44:46.294
+Compiling in "/home/..../test/property_test"
+  Deleted:   ["ct_prop.beam"]
+  ErlFiles:  ["ct_prop.erl"]
+  MacroDefs: [{d,'PROPER'}]
+
+Testing lib.common_test.ct_property_test_SUITE: TEST COMPLETE, 1 ok, 0 failed of 1 test cases
+
+....
+
+
+ + +
+ + A stateful testing example +

Assume a test that generates some parallel stateful commands, and runs 300 tests:

+ +prop_parallel(Config) -> + numtests(300, + ?FORALL(Cmds, parallel_commands(?MODULE), + begin + RunResult = run_parallel_commands(?MODULE, Cmds), + ct_property_test:present_result(?MODULE, Cmds, RunResult, Config) + end)). + +

The + ct_property_test:present_result/4 + is a help function for printing some statistics in the CommonTest log file.

+

Our example test could for example be a simple test of an ftp server, where we perform get, put + and delete requests, some of them in parallel. Per default, the result has three sections: +

+ 
+*** User 2019-12-11 13:28:17.504 ***
+
+Distribution sequential/parallel
+
+ 57.7% sequential
+ 28.0% parallel_2
+ 14.3% parallel_1
+
+
+
+*** User 2019-12-11 13:28:17.505 ***
+
+Function calls
+
+ 44.4% get
+ 39.3% put
+ 16.3% delete
+
+
+
+*** User 2019-12-11 13:28:17.505 ***
+
+Length of command sequences
+
+Range  : Number in range
+-------:----------------
+ 0 -  4:    8    2.7%  <-- min=3
+ 5 -  9:   44   14.7% 
+10 - 14:   74   24.7% 
+15 - 19:   60   20.0%  <-- mean=18.7 <-- median=16.0
+20 - 24:   38   12.7% 
+25 - 29:   26    8.7% 
+30 - 34:   19    6.3% 
+35 - 39:   19    6.3% 
+40 - 44:    8    2.7% 
+45 - 49:    4    1.3%  <-- max=47
+        ------
+          300
+
+

The first part - Distribution sequential/parallel - shows the distribution in the + sequential and parallel part of the result of parallel_commands/1. See any property testing tool for + an explanation of this function. + The table shows that of all commands (get and put in our case), + 57.7% are executed in the sequential part prior to the parallel part, + 28.0% are executed in the first parallel list and the rest in the second parallel list. +

+ +

The second part - Function calls - shows the distribution of the three calls in the + generated command lists. We see that all of the three calls are executed. If it was so that we + thought that we also generated a fourth call, a table like this shows that we failed with that. +

+ +

The third and final part - Length of command sequences - show statistics of the + generated command sequences. We see that the shortest list has three elementes while the longest + has 47 elements. The mean and median values are also shown. Further we could for example see that + only 2.7% of the lists (that is eight lists) only has three or four elements. +

+ +
+ + + diff --git a/lib/common_test/doc/src/part.xml b/lib/common_test/doc/src/part.xml index 000eb06b82..66dcf75258 100644 --- a/lib/common_test/doc/src/part.xml +++ b/lib/common_test/doc/src/part.xml @@ -48,6 +48,7 @@ + -- cgit v1.2.1 From f1869655e76e876b19201095b104b394b8faea36 Mon Sep 17 00:00:00 2001 From: Hans Nilsson Date: Wed, 8 Jan 2020 18:05:38 +0100 Subject: common_test: Comments from doc review --- lib/common_test/doc/src/ct_property_test.xml | 7 +++++-- lib/common_test/doc/src/ct_property_test_chapter.xml | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/common_test/doc/src/ct_property_test.xml b/lib/common_test/doc/src/ct_property_test.xml index 687da15b0a..1690e9962a 100644 --- a/lib/common_test/doc/src/ct_property_test.xml +++ b/lib/common_test/doc/src/ct_property_test.xml @@ -123,7 +123,9 @@ prop_ftp() -> property_test. All found Erlang files in that directory are compiled with one of the macros 'EQC', 'PROPER' or 'TRIQ' set, depending on which tool - that is first found. + that is first found. This could make parts of the Erlang property tests + code to be included or excluded with the macro directives + -ifdef(Macro). or -ifndef(Macro)..

The file(s) in the property_test subdirectory could, or should, include the ct_property_test include file: @@ -136,7 +138,8 @@ prop_ftp() -> Include the correct tool's include file Set the macro 'MOD_eqc' to the correct module name for the - selected tool, that is eqc, proper or triq. + selected tool. That is, the macro 'MOD_eqc' is set to either + eqc, proper or triq. diff --git a/lib/common_test/doc/src/ct_property_test_chapter.xml b/lib/common_test/doc/src/ct_property_test_chapter.xml index 8cc273d2fc..131f3a962d 100644 --- a/lib/common_test/doc/src/ct_property_test_chapter.xml +++ b/lib/common_test/doc/src/ct_property_test_chapter.xml @@ -71,12 +71,12 @@

Assume that we want to test the lists:sort/1 function.

We need a property to test the function. In normal way, we create - property_test/test_sort.erl module in the test directory + property_test/ct_prop.erl module in the test directory in our application:

--module(test_sort). +-module(ct_prop). -export([prop_sort/0]). %%% This will include the .hrl file for the installed testing tool: -- cgit v1.2.1