Merge branch 'traveltissues/strictxfail' into 'master'

setup.cfg: make xfails strict by default

See merge request BuildStream/buildstream!1632

11 files changed, 19 insertions, 18 deletions

diff --git a/setup.cfg b/setup.cfg
index a8919fdca..2264a317c 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -21,6 +21,7 @@ markers =
     integration: run test only if --integration option is specified
     remoteexecution: run test only if --remote-execution option is specified
     in_subprocess: run test in a Python process forked from the main one
+xfail_strict=True
 
 [pycodestyle]
 max-line-length = 119
diff --git a/tests/elements/filter.py b/tests/elements/filter.py
index af3f348b2..99370052e 100644
--- a/tests/elements/filter.py
+++ b/tests/elements/filter.py
@@ -31,7 +31,7 @@ def test_filter_include(datafiles, cli, tmpdir):
     assert not os.path.exists(os.path.join(checkout, "bar"))
 
 
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
 def test_filter_include_dynamic(datafiles, cli, tmpdir):
     project = str(datafiles)
diff --git a/tests/examples/autotools.py b/tests/examples/autotools.py
index ecd140718..45783b30c 100644
--- a/tests/examples/autotools.py
+++ b/tests/examples/autotools.py
@@ -43,7 +43,7 @@ def test_autotools_build(cli, datafiles):
 @pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
                     reason='Examples are written for x86-64')
 @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 @pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
 @pytest.mark.datafiles(DATA_DIR)
 def test_autotools_run(cli, datafiles):
diff --git a/tests/frontend/configurable_warnings.py b/tests/frontend/configurable_warnings.py
index 38b7f2912..caa91bb61 100644
--- a/tests/frontend/configurable_warnings.py
+++ b/tests/frontend/configurable_warnings.py
@@ -46,7 +46,6 @@ def build_project(datafiles, fatal_warnings):
     return project_path
 
 
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 @pytest.mark.datafiles(TOP_DIR)
 @pytest.mark.parametrize("element_name, fatal_warnings, expect_fatal, error_domain", [
     ("corewarn.bst", [CoreWarnings.OVERLAPS], True, ErrorDomain.STREAM),
@@ -60,6 +59,8 @@ def build_project(datafiles, fatal_warnings):
 ])
 def test_fatal_warnings(cli, datafiles, element_name,
                         fatal_warnings, expect_fatal, error_domain):
+    if HAVE_SANDBOX == 'buildbox' and error_domain != ErrorDomain.STREAM:
+        pytest.xfail()
     project_path = build_project(datafiles, fatal_warnings)
 
     result = cli.run(project=project_path, args=["build", element_name])
diff --git a/tests/integration/autotools.py b/tests/integration/autotools.py
index c985e8e6c..16cb38af7 100644
--- a/tests/integration/autotools.py
+++ b/tests/integration/autotools.py
@@ -44,7 +44,7 @@ def test_autotools_build(cli, datafiles):
 # amhello project for this.
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 def test_autotools_confroot_build(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
diff --git a/tests/integration/cachedfail.py b/tests/integration/cachedfail.py
index e3c05e691..63ad2d4d3 100644
--- a/tests/integration/cachedfail.py
+++ b/tests/integration/cachedfail.py
@@ -139,7 +139,6 @@ def test_build_depend_on_cached_fail(cli, datafiles):
 
 
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.parametrize("on_error", ("continue", "quit"))
 def test_push_cached_fail(cli, tmpdir, datafiles, on_error):
diff --git a/tests/integration/cmake.py b/tests/integration/cmake.py
index 0fe91ed6d..a0298c2c3 100644
--- a/tests/integration/cmake.py
+++ b/tests/integration/cmake.py
@@ -36,7 +36,7 @@ def test_cmake_build(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 def test_cmake_confroot_build(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
diff --git a/tests/integration/script.py b/tests/integration/script.py
index dc893f679..0e88ae53c 100644
--- a/tests/integration/script.py
+++ b/tests/integration/script.py
@@ -100,7 +100,7 @@ def test_script_root(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 def test_script_no_root(cli, datafiles):
     project = str(datafiles)
     element_path = os.path.join(project, 'elements')
@@ -124,7 +124,7 @@ def test_script_no_root(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 def test_script_cwd(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
diff --git a/tests/integration/shell.py b/tests/integration/shell.py
index b0de76ef5..a44f51609 100644
--- a/tests/integration/shell.py
+++ b/tests/integration/shell.py
@@ -177,7 +177,7 @@ def test_no_shell(cli, datafiles):
 @pytest.mark.parametrize("path", [("/etc/pony.conf"), ("/usr/share/pony/pony.txt")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 def test_host_files(cli, datafiles, path):
     project = str(datafiles)
     ponyfile = os.path.join(project, 'files', 'shell-mount', 'pony.txt')
@@ -199,7 +199,7 @@ def test_host_files(cli, datafiles, path):
 @pytest.mark.parametrize("path", [("/etc"), ("/usr/share/pony")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 def test_host_files_expand_environ(cli, datafiles, path):
     project = str(datafiles)
     hostpath = os.path.join(project, 'files', 'shell-mount')
@@ -282,7 +282,7 @@ def test_host_files_missing(cli, datafiles, optional):
 @pytest.mark.parametrize("path", [("/etc/pony.conf"), ("/usr/share/pony/pony.txt")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 def test_cli_mount(cli, datafiles, path):
     project = str(datafiles)
     ponyfile = os.path.join(project, 'files', 'shell-mount', 'pony.txt')
@@ -329,7 +329,7 @@ def test_workspace_visible(cli, datafiles):
 # Test that '--sysroot' works
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 def test_sysroot(cli, tmpdir, datafiles):
     project = str(datafiles)
     base_element = "base/base-alpine.bst"
diff --git a/tests/integration/workspace.py b/tests/integration/workspace.py
index a4cb454d1..8529fc4e1 100644
--- a/tests/integration/workspace.py
+++ b/tests/integration/workspace.py
@@ -51,8 +51,8 @@ def test_workspace_mount_on_read_only_directory(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
-@pytest.mark.xfail(reason="Incremental builds are currently incompatible with workspace source plugin.", strict=True)
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
+@pytest.mark.xfail(reason="Incremental builds are currently incompatible with workspace source plugin.")
 def test_workspace_commanddir(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -252,8 +252,8 @@ def test_updated_dependency_nested(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
-@pytest.mark.xfail(reason="Incremental builds are currently incompatible with workspace source plugin.", strict=True)
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
+@pytest.mark.xfail(reason="Incremental builds are currently incompatible with workspace source plugin.")
 def test_incremental_configure_commands_run_only_once(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -306,7 +306,7 @@ def test_incremental_configure_commands_run_only_once(cli, datafiles):
 #
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 def test_workspace_missing_last_successful(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
diff --git a/tests/sourcecache/push.py b/tests/sourcecache/push.py
index 1be2d40cd..406aeba9f 100644
--- a/tests/sourcecache/push.py
+++ b/tests/sourcecache/push.py
@@ -260,7 +260,7 @@ def test_push_fail(cli, tmpdir, datafiles):
     assert "Pushed" not in res.stderr
 
 
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 @pytest.mark.datafiles(DATA_DIR)
 def test_source_push_build_fail(cli, tmpdir, datafiles):
     project_dir = str(datafiles)