diff options
Diffstat (limited to 'lib/common_test/src/test_server_ctrl.erl')
| -rw-r--r-- | lib/common_test/src/test_server_ctrl.erl | 183 |
1 files changed, 161 insertions, 22 deletions
diff --git a/lib/common_test/src/test_server_ctrl.erl b/lib/common_test/src/test_server_ctrl.erl index 9412c43187..3bb7b6263c 100644 --- a/lib/common_test/src/test_server_ctrl.erl +++ b/lib/common_test/src/test_server_ctrl.erl @@ -89,6 +89,7 @@ -define(logdir_ext, ".logs"). -define(data_dir_suffix, "_data/"). -define(suitelog_name, "suite.log"). +-define(suitelog_latest_name, "suite.log.latest"). -define(coverlog_name, "cover.html"). -define(raw_coverlog_name, "cover.log"). -define(cross_coverlog_name, "cross_cover.html"). @@ -1126,6 +1127,7 @@ init_tester(Mod, Func, Args, Dir, Name, {_,_,MinLev}=Levels, RejectIoReqs, CreatePrivDir, TCCallback, ExtraTools) -> process_flag(trap_exit, true), _ = test_server_io:start_link(), + put(app, common_test), put(test_server_name, Name), put(test_server_dir, Dir), put(test_server_total_time, 0), @@ -1150,6 +1152,12 @@ init_tester(Mod, Func, Args, Dir, Name, {_,_,MinLev}=Levels, end, %% before first print, read and set logging options + FWLogDir = + case test_server_sup:framework_call(get_log_dir, [], []) of + {ok,FwDir} -> FwDir; + _ -> filename:dirname(Dir) + end, + put(test_server_framework_logdir, FWLogDir), LogOpts = test_server_sup:framework_call(get_logopts, [], []), put(test_server_logopts, LogOpts), @@ -1435,6 +1443,8 @@ remove_conf([C={Mod,error_in_suite,_}|Cases], NoConf, Repeats) -> true -> remove_conf(Cases, [C|NoConf], Repeats) end; +remove_conf([C={repeat,_,_}|Cases], NoConf, _Repeats) -> + remove_conf(Cases, [C|NoConf], true); remove_conf([C|Cases], NoConf, Repeats) -> remove_conf(Cases, [C|NoConf], Repeats); remove_conf([], NoConf, true) -> @@ -1711,6 +1721,12 @@ start_log_file() -> test_server_io:set_fd(html, Html), test_server_io:set_fd(unexpected_io, Unexpected), + %% we must assume the redirection file (to the latest suite index) can + %% be stored on the level above the log directory of the current test + TopDir = filename:dirname(get(test_server_framework_logdir)), + RedirectLink = filename:join(TopDir, ?suitelog_latest_name ++ ?html_ext), + make_html_link(RedirectLink, HtmlName, redirect), + make_html_link(filename:absname(?last_test ++ ?html_ext), HtmlName, filename:basename(Dir)), LinkName = filename:join(Dir, ?last_link), @@ -1739,11 +1755,18 @@ make_html_link(LinkName, Target, Explanation) -> false -> "file:" ++ uri_encode(Target) end, - H = [html_header(Explanation), - "<h1>Last test</h1>\n" - "<a href=\"",Href,"\">",Explanation,"</a>\n" - "</body>\n</html>\n"], + H = if Explanation == redirect -> + Meta = ["<meta http-equiv=\"refresh\" " + "content=\"0; url=", Href, "\" />\n"], + [html_header("redirect", Meta), "</html>\n"]; + true -> + [html_header(Explanation), + "<h1>Last test</h1>\n" + "<a href=\"",Href,"\">",Explanation,"</a>\n" + "</body>\n</html>\n"] + end, ok = write_html_file(LinkName, H). + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% start_minor_log_file(Mod, Func, ParallelTC) -> AbsName @@ -2039,6 +2062,14 @@ add_init_and_end_per_suite([SkipCase|Cases], LastMod, LastRef, FwMod) [SkipCase|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; add_init_and_end_per_suite([{conf,_,_,_}=Case|Cases], LastMod, LastRef, FwMod) -> [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; +add_init_and_end_per_suite([{repeat,{Mod,_},_}=Case|Cases], LastMod, LastRef, FwMod) + when Mod =/= LastMod, Mod =/= FwMod -> + {PreCases, NextMod, NextRef} = + do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod), + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, + NextRef, FwMod)]; +add_init_and_end_per_suite([{repeat,_,_}=Case|Cases], LastMod, LastRef, FwMod) -> + [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; add_init_and_end_per_suite([{Mod,_}=Case|Cases], LastMod, LastRef, FwMod) when Mod =/= LastMod, Mod =/= FwMod -> {PreCases, NextMod, NextRef} = @@ -2116,7 +2147,7 @@ do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) -> %% let's call a "fake" end_per_suite if it exists case erlang:function_exported(FwMod, end_per_suite, 1) of true -> - [{conf,LastRef,[{suite,Mod}], + [{conf,LastRef,[{suite,LastMod}], {FwMod,end_per_suite}}|Init]; false -> [{conf,LastRef,[],{LastMod,end_per_suite}}|Init] @@ -2163,6 +2194,7 @@ do_add_end_per_suite_and_skip(LastMod, LastRef, Mod, FwMod) -> %% Runs the specified tests, then displays/logs the summary. run_test_cases(TestSpec, Config, TimetrapData) -> + test_server:init_valgrind(), case lists:member(no_src, get(test_server_logopts)) of true -> ok; @@ -2903,6 +2935,29 @@ run_test_cases_loop([{conf,_Ref,_Props,_X}=Conf|_Cases0], Config, _TimetrapData, _Mode, _Status) -> erlang:error(badarg, [Conf,Config]); +run_test_cases_loop([{repeat,Case,{RepeatType,N}}|Cases0], Config, + TimeTrapData, Mode, Status) -> + Ref = make_ref(), + Parallel = check_prop(parallel, Mode) =/= false, + Sequence = check_prop(sequence, Mode) =/= false, + RepeatStop = RepeatType=:=repeat_until_fail + orelse RepeatType=:=repeat_until_ok, + + if Parallel andalso RepeatStop -> + %% Cannot check results of test case during parallal + %% execution, so only RepeatType=:=repeat is allowed in + %% combination with parallel groups. + erlang:error({illegal_combination,{parallel,RepeatType}}); + Sequence andalso RepeatStop -> + %% Sequence is stop on fail + skip rest, so only + %% RepeatType=:=repeat makes sense inside a sequence. + erlang:error({illegal_combination,{sequence,RepeatType}}); + true -> + Mode1 = [{Ref,[{repeat,{RepeatType,1,N}}],?now}|Mode], + run_test_cases_loop([Case | Cases0], Config, TimeTrapData, + Mode1, Status) + end; + run_test_cases_loop([{Mod,Case}|Cases], Config, TimetrapData, Mode, Status) -> ActualCfg = case get(test_server_create_priv_dir) of @@ -2915,7 +2970,7 @@ run_test_cases_loop([{Mod,Case}|Cases], Config, TimetrapData, Mode, Status) -> run_test_cases_loop([{Mod,Case,[ActualCfg]}|Cases], Config, TimetrapData, Mode, Status); -run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status) -> +run_test_cases_loop([{Mod,Func,Args}=Case|Cases], Config, TimetrapData, Mode0, Status) -> {Num,RunInit} = case FwMod = get_fw_mod(?MODULE) of Mod when Func == error_in_suite -> @@ -2925,6 +2980,14 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status) run_init} end, + Mode = + case Mode0 of + [{_,[{repeat,{_,_,_}}],_}|RestMode] -> + RestMode; + _ -> + Mode0 + end, + %% check the current execution mode and save info about the case if %% detected that printouts to common log files is handled later @@ -2952,36 +3015,42 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status) if is_tuple(RetVal) -> element(1,RetVal); true -> undefined end, - {Failed,Status1} = + {Result,Failed,Status1} = case RetTag of Skip when Skip==skip; Skip==skipped -> - {false,update_status(skipped, Mod, Func, Status)}; + {skipped,false,update_status(skipped, Mod, Func, Status)}; Fail when Fail=='EXIT'; Fail==failed -> - {true,update_status(failed, Mod, Func, Status)}; + {failed,true,update_status(failed, Mod, Func, Status)}; _ when Time==died, RetVal=/=ok -> - {true,update_status(failed, Mod, Func, Status)}; + {failed,true,update_status(failed, Mod, Func, Status)}; _ -> - {false,update_status(ok, Mod, Func, Status)} + {ok,false,update_status(ok, Mod, Func, Status)} end, case check_prop(sequence, Mode) of false -> + {Cases1,Mode1} = + check_repeat_testcase(Case,Result,Cases,Mode0), stop_minor_log_file(), - run_test_cases_loop(Cases, Config, TimetrapData, Mode, Status1); + run_test_cases_loop(Cases1, Config, TimetrapData, Mode1, Status1); Ref -> %% the case is in a sequence; we must check the result and %% determine if the following cases should run or be skipped if not Failed -> % proceed with next case + {Cases1,Mode1} = + check_repeat_testcase(Case,Result,Cases,Mode0), stop_minor_log_file(), - run_test_cases_loop(Cases, Config, TimetrapData, Mode, Status1); + run_test_cases_loop(Cases1, Config, TimetrapData, Mode1, Status1); true -> % skip rest of cases in sequence print(minor, "~n*** ~tw failed.~n" " Skipping all other cases in sequence.", [Func]), + {Cases1,Mode1} = + check_repeat_testcase(Case,Result,Cases,Mode0), Reason = {failed,{Mod,Func}}, - Cases2 = skip_cases_upto(Ref, Cases, Reason, tc, + Cases2 = skip_cases_upto(Ref, Cases1, Reason, tc, Mode, auto_skip_case), stop_minor_log_file(), - run_test_cases_loop(Cases2, Config, TimetrapData, Mode, Status1) + run_test_cases_loop(Cases2, Config, TimetrapData, Mode1, Status1) end end; %% the test case is being executed in parallel with the main process (and @@ -2990,7 +3059,8 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status) %% io from Pid will be buffered by the test_server_io process and %% handled later, so we have to save info about the case queue_test_case_io(undefined, Pid, Num+1, Mod, Func), - run_test_cases_loop(Cases, Config, TimetrapData, Mode, Status) + {Cases1,Mode1} = check_repeat_testcase(Case,ok,Cases,Mode0), + run_test_cases_loop(Cases1, Config, TimetrapData, Mode1, Status) end; %% TestSpec processing finished @@ -3428,9 +3498,19 @@ modify_cases_upto1(Ref, {skip,Reason,FType,Mode,SkipType}, T, Orig, Alt) end; -%% next is some other case, ignore or copy -modify_cases_upto1(Ref, {skip,_,_,_,_}=Op, [_Other|T], Orig, Alt) -> +%% next is a repeated test case +modify_cases_upto1(Ref, {skip,Reason,_,Mode,SkipType}=Op, + [{repeat,{_M,_F}=MF,_Repeat}|T], Orig, Alt) -> + modify_cases_upto1(Ref, Op, T, Orig, [{SkipType,{MF,Reason},Mode}|Alt]); + +%% next is an already skipped case, ignore or copy +modify_cases_upto1(Ref, {skip,_,_,_,_}=Op, [{SkipType,_,_}|T], Orig, Alt) + when SkipType=:=skip_case; SkipType=:=auto_skip_case -> modify_cases_upto1(Ref, Op, T, Orig, Alt); + +%% next is some other case, mark as skipped or copy +modify_cases_upto1(Ref, {skip,Reason,_,Mode,SkipType}=Op, [Other|T], Orig, Alt) -> + modify_cases_upto1(Ref, Op, T, Orig, [{SkipType,{Other,Reason},Mode}|Alt]); modify_cases_upto1(Ref, CopyOp, [C|T], Orig, Alt) -> modify_cases_upto1(Ref, CopyOp, T, [C|Orig], [C|Alt]). @@ -3703,6 +3783,7 @@ run_test_case(Ref, Num, Mod, Func, Args, RunInit, TimetrapData, Mode) -> spawn_link( fun() -> process_flag(trap_exit, true), + ct_util:mark_process(), _ = [put(Key, Val) || {Key,Val} <- Dictionary], set_io_buffering({tc,Main}), run_test_case1(Ref, Num, Mod, Func, Args, RunInit, @@ -3796,7 +3877,7 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, %% run the test case {Result,DetectedFail,ProcsBefore,ProcsAfter} = - run_test_case_apply(Mod, Func, [UpdatedArgs], GrName, + run_test_case_apply(Num, Mod, Func, [UpdatedArgs], GrName, RunInit, TimetrapData), {Time,RetVal,Loc,Opts,Comment} = case Result of @@ -3817,6 +3898,10 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, {died,{timetrap_timeout,TimetrapTimeout}} -> progress(failed, Num, Mod, Func, GrName, Loc, timetrap_timeout, TimetrapTimeout, Comment, Style); + {died,Reason={auto_skip,_Why}} -> + %% died in init_per_testcase or in a hook in this context + progress(skip, Num, Mod, Func, GrName, Loc, Reason, + Time, Comment, Style); {died,{Skip,Reason}} when Skip==skip; Skip==skipped -> %% died in init_per_testcase progress(skip, Num, Mod, Func, GrName, Loc, Reason, @@ -4366,7 +4451,7 @@ do_format_exception(Reason={Error,Stack}) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% run_test_case_apply(Mod, Func, Args, Name, RunInit, +%% run_test_case_apply(CaseNum, Mod, Func, Args, Name, RunInit, %% TimetrapData) -> %% {{Time,RetVal,Loc,Opts,Comment},DetectedFail,ProcessesBefore,ProcessesAfter} | %% {{died,Reason,unknown,Comment},DetectedFail,ProcessesBefore,ProcessesAfter} @@ -4380,9 +4465,9 @@ do_format_exception(Reason={Error,Stack}) -> %% ProcessesBefore = ProcessesAfter = integer() %% -run_test_case_apply(Mod, Func, Args, Name, RunInit, +run_test_case_apply(CaseNum, Mod, Func, Args, Name, RunInit, TimetrapData) -> - test_server:run_test_case_apply({Mod,Func,Args,Name,RunInit, + test_server:run_test_case_apply({CaseNum,Mod,Func,Args,Name,RunInit, TimetrapData}). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -4772,6 +4857,14 @@ collect_cases({make,InitMFA,CaseList,FinMFA}, St0, Mode) -> {error,_Reason} = Error -> Error end; +collect_cases({repeat,{Module, Case}, Repeat}, St, Mode) -> + case catch collect_case([Case], St#cc{mod=Module}, [], Mode) of + {ok, [{Module,Case}], _} -> + {ok, [{repeat,{Module, Case}, Repeat}], St}; + Other -> + {error,Other} + end; + collect_cases({Module, Cases}, St, Mode) when is_list(Cases) -> case (catch collect_case(Cases, St#cc{mod=Module}, [], Mode)) of Result = {ok,_,_} -> @@ -5654,6 +5747,13 @@ html_header(Title) -> "<body bgcolor=\"white\" text=\"black\" " "link=\"blue\" vlink=\"purple\" alink=\"red\">\n"]. +html_header(Title, Meta) -> + ["<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 3.2 Final//EN\">\n" + "<!-- autogenerated by '", atom_to_list(?MODULE), "'. -->\n" + "<html>\n" + "<head>\n" + "<title>", Title, "</title>\n"] ++ Meta ++ ["</head>\n"]. + open_html_file(File) -> open_utf8_file(File). @@ -5728,3 +5828,42 @@ encoding(File) -> E -> E end. + +check_repeat_testcase(Case,Result,Cases, + [{Ref,[{repeat,RepeatData0}],StartTime}|Mode0]) -> + case do_update_repeat_data(Result,RepeatData0) of + false -> + {Cases,Mode0}; + RepeatData -> + {[Case|Cases],[{Ref,[{repeat,RepeatData}],StartTime}|Mode0]} + end; +check_repeat_testcase(_,_,Cases,Mode) -> + {Cases,Mode}. + +do_update_repeat_data(_,{RT,N,N}) when is_integer(N) -> + report_repeat_testcase(N,N), + report_stop_repeat_testcase(done,{RT,N}), + false; +do_update_repeat_data(ok,{repeat_until_ok=RT,M,N}) -> + report_repeat_testcase(M,N), + report_stop_repeat_testcase(RT,{RT,N}), + false; +do_update_repeat_data(failed,{repeat_until_fail=RT,M,N}) -> + report_repeat_testcase(M,N), + report_stop_repeat_testcase(RT,{RT,N}), + false; +do_update_repeat_data(_,{RT,M,N}) when is_integer(M) -> + report_repeat_testcase(M,N), + {RT,M+1,N}; +do_update_repeat_data(_,{_,M,N}=RepeatData) -> + report_repeat_testcase(M,N), + RepeatData. + +report_stop_repeat_testcase(Reason,RepVal) -> + print(minor, "~n*** Stopping test case repeat operation: ~w", [Reason]), + print(1, "Stopping test case repeat operation: ~w", [RepVal]). + +report_repeat_testcase(M,forever) -> + print(minor, "~n=== Repeated test case: ~w of infinity", [M]); +report_repeat_testcase(M,N) -> + print(minor, "~n=== Repeated test case: ~w of ~w", [M,N]). |