Additional Common Test regression tests

One test that checks for a failed make, another
that detects a failed test run when only one of several
suites fails. The ct test fails is only applicable to OTP
versions older than R16 (inclusive), this is related to the way
older versions handle the test .spec files.

10 files changed, 120 insertions, 0 deletions

diff --git a/inttest/ct_make_fails/app.config b/inttest/ct_make_fails/app.config
new file mode 100644
index 0000000..bb718b2
--- /dev/null
+++ b/inttest/ct_make_fails/app.config
@@ -0,0 +1,2 @@
+%% This file is an application config file, not a CT test config file
+[{a1, [{foo, bar}]}].
diff --git a/inttest/ct_make_fails/ct_make_fails_rt.erl b/inttest/ct_make_fails/ct_make_fails_rt.erl
new file mode 100644
index 0000000..a7959a7
--- /dev/null
+++ b/inttest/ct_make_fails/ct_make_fails_rt.erl
@@ -0,0 +1,31 @@
+%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 et
+-module(ct_make_fails_rt).
+
+-compile(export_all).
+
+
+files() ->
+    [{create, "ebin/a1.app", app(a1)},
+     {copy, "../../rebar", "rebar"},
+     {copy, "rebar.config", "rebar.config"},
+     {copy, "app.config", "app.config"},
+     {copy, "test_SUITE.erl", "itest/test_SUITE.erl"}].
+
+run(_Dir) ->
+    ok = case catch retest:sh("./rebar compile ct -v") of
+                {error, {stopped, _}} -> ok;
+                _ -> expected_to_fail
+              end.
+
+%%
+%% Generate the contents of a simple .app file
+%%
+app(Name) ->
+    App = {application, Name,
+           [{description, atom_to_list(Name)},
+            {vsn, "1"},
+            {modules, []},
+            {registered, []},
+            {applications, [kernel, stdlib]}]},
+    io_lib:format("~p.\n", [App]).
diff --git a/inttest/ct_make_fails/rebar.config b/inttest/ct_make_fails/rebar.config
new file mode 100644
index 0000000..58047ba
--- /dev/null
+++ b/inttest/ct_make_fails/rebar.config
@@ -0,0 +1,2 @@
+{ct_dir, "itest"}.
+{ct_extra_params, "-repeat 2 -erl_args -config app"}.
diff --git a/inttest/ct_make_fails/test_SUITE.erl b/inttest/ct_make_fails/test_SUITE.erl
new file mode 100644
index 0000000..343aa5a
--- /dev/null
+++ b/inttest/ct_make_fails/test_SUITE.erl
@@ -0,0 +1,17 @@
+-module(test_SUITE).
+
+-compile(export_all).
+
+-include_lib("ct.hrl").
+
+all() ->
+    [simple_test,
+     app_config_file_test].
+
+simple_test(Config) ->
+    io:format("Test: ~p\n" [Config]).
+
+app_config_file_test(_Config) ->
+    application:start(a1),
+    {ok, bar} = application:get_env(a1, foo),
+    application:stop(a1).
diff --git a/inttest/ct_test_fails/app.config b/inttest/ct_test_fails/app.config
new file mode 100644
index 0000000..bb718b2
--- /dev/null
+++ b/inttest/ct_test_fails/app.config
@@ -0,0 +1,2 @@
+%% This file is an application config file, not a CT test config file
+[{a1, [{foo, bar}]}].
diff --git a/inttest/ct_test_fails/ct_test_fails.test.spec b/inttest/ct_test_fails/ct_test_fails.test.spec
new file mode 100644
index 0000000..7bba687
--- /dev/null
+++ b/inttest/ct_test_fails/ct_test_fails.test.spec
@@ -0,0 +1 @@
+{suites,"../itest",[test_SUITE, test2_SUITE]}.
diff --git a/inttest/ct_test_fails/ct_test_fails_rt.erl b/inttest/ct_test_fails/ct_test_fails_rt.erl
new file mode 100644
index 0000000..a556891
--- /dev/null
+++ b/inttest/ct_test_fails/ct_test_fails_rt.erl
@@ -0,0 +1,33 @@
+%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 et
+-module(ct_test_fails_rt).
+
+-compile(export_all).
+
+
+files() ->
+    [{create, "ebin/a1.app", app(a1)},
+     {copy, "../../rebar", "rebar"},
+     {copy, "rebar.config", "rebar.config"},
+     {copy, "app.config", "app.config"},
+     {copy, "ct_test_fails.test.spec", "itest/ct_test_fails.test.spec"},
+     {copy, "test_SUITE.erl", "itest/test_SUITE.erl"},
+     {copy, "test2_SUITE.erl", "itest/test2_SUITE.erl"}].
+
+run(Dir) ->
+    ok = case catch retest:sh("./rebar compile ct -v 3") of
+                {error, {stopped, _}} -> ok;
+                _ -> expected_to_fail
+              end.
+
+%%
+%% Generate the contents of a simple .app file
+%%
+app(Name) ->
+    App = {application, Name,
+           [{description, atom_to_list(Name)},
+            {vsn, "1"},
+            {modules, []},
+            {registered, []},
+            {applications, [kernel, stdlib]}]},
+    io_lib:format("~p.\n", [App]).
diff --git a/inttest/ct_test_fails/rebar.config b/inttest/ct_test_fails/rebar.config
new file mode 100644
index 0000000..84c23ef
--- /dev/null
+++ b/inttest/ct_test_fails/rebar.config
@@ -0,0 +1,13 @@
+{ct_dir, ["itest"]}.
+{ct_extra_params, "-erl_args -config app"}.
+
+%% http://erlang.org/doc/apps/common_test/run_test_chapter.html#id77160
+%%      Any relative paths specified in the test specification, will be relative to the
+%%      directory which contains the  test specification file, if ct_run -spec TestSpecFile ...
+%%      or ct:run:test([{spec,TestSpecFile},...]) executes the  test. The path will be
+%%      relative to the top level log directory, if ct:run:testspec(TestSpec) executes the test.
+%% however for versions older than R16  what counts is the project root path and not the path
+%% of the location of the test spec. This will cause the test to fail since R15/14 can't find the
+%% test.spec file. Since we can't change the file we have no choice but to bypass the test
+%% completely
+{require_min_otp_vsn, "R16B"}.
diff --git a/inttest/ct_test_fails/test2_SUITE.erl b/inttest/ct_test_fails/test2_SUITE.erl
new file mode 100644
index 0000000..4b9299f
--- /dev/null
+++ b/inttest/ct_test_fails/test2_SUITE.erl
@@ -0,0 +1,9 @@
+-module(test2_SUITE).
+
+-compile(export_all).
+
+all() ->
+    [simple_test].
+
+simple_test(Config) ->
+    io:format("Test: ~p\n", [Config]).
diff --git a/inttest/ct_test_fails/test_SUITE.erl b/inttest/ct_test_fails/test_SUITE.erl
new file mode 100644
index 0000000..124fbe8
--- /dev/null
+++ b/inttest/ct_test_fails/test_SUITE.erl
@@ -0,0 +1,10 @@
+-module(test_SUITE).
+
+-compile(export_all).
+
+all() ->
+    [simple_test].
+
+simple_test(Config) ->
+    io:format("Test: ~p\n", [Config]),
+    ok = not_ok.