Testbschubert/test-skips

author: Benjamin Schubert <contact@benschubert.me> 2019-10-09 10:56:29 +0100
committer: Benjamin Schubert <contact@benschubert.me> 2019-10-09 10:56:29 +0100
commit: b1b0d28e5051a78cd7b04fe70ed9dffb1f62069f (patch)
tree: 3fcb14c9b33ead2cd5913498b0dfce58247ae78d
parent: b592cb85c7e8f3580e75dff531d44ec1bed84dc1 (diff)
download: buildstream-bschubert/test-skips.tar.gz
47 files changed, 180 insertions, 180 deletions
diff --git a/setup.cfg b/setup.cfg
index a8919fdca..8a283f9c1 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -16,7 +16,7 @@ norecursedirs = tests/integration/project integration-cache tmp __pycache__ .egg
 python_files = tests/*/*.py
 env =
     D:BST_TEST_SUITE=True
-
+xfail_strict = True
 markers =
     integration: run test only if --integration option is specified
     remoteexecution: run test only if --remote-execution option is specified
diff --git a/src/buildstream/testing/_sourcetests/source_determinism.py b/src/buildstream/testing/_sourcetests/source_determinism.py
index fc0e4618c..b803ce67d 100644
--- a/src/buildstream/testing/_sourcetests/source_determinism.py
+++ b/src/buildstream/testing/_sourcetests/source_determinism.py
@@ -49,8 +49,8 @@ def create_test_directory(*path, mode=0o644):
 
 @pytest.mark.integration
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.skipif(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox, Must Fix')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox, Must Fix')
 def test_deterministic_source_umask(cli, tmpdir, datafiles, kind):
     project = str(datafiles)
     element_name = 'list.bst'
diff --git a/src/buildstream/testing/_utils/junction.py b/src/buildstream/testing/_utils/junction.py
index 98d23b0a2..3bb6f03ee 100644
--- a/src/buildstream/testing/_utils/junction.py
+++ b/src/buildstream/testing/_utils/junction.py
@@ -43,7 +43,7 @@ def generate_junction(tmpdir, subproject_path, junction_path, *, store_ref=True)
 class _SimpleGit(Repo):
     def __init__(self, directory, subdir='repo'):
         if not HAVE_GIT:
-            pytest.skip('git is not available')
+            pytest.xfail('git is not available')
         super().__init__(directory, subdir)
 
     def create(self, directory):
diff --git a/tests/cachekey/cachekey.py b/tests/cachekey/cachekey.py
index acfe97947..af20fb50a 100644
--- a/tests/cachekey/cachekey.py
+++ b/tests/cachekey/cachekey.py
@@ -150,11 +150,11 @@ DATA_DIR = os.path.join(
 # The cache key test uses a project which exercises all plugins,
 # so we cant run it at all if we dont have them installed.
 #
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
+@pytest.mark.xfail(MACHINE_ARCH != 'x86-64',
                     reason='Cache keys depend on architecture')
-@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
-@pytest.mark.skipif(HAVE_BZR is False, reason="bzr is not available")
-@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.xfail(not IS_LINUX, reason='Only available on linux')
+@pytest.mark.xfail(HAVE_BZR is False, reason="bzr is not available")
+@pytest.mark.xfail(HAVE_GIT is False, reason="git is not available")
 @pytest.mark.datafiles(DATA_DIR)
 def test_cache_key(datafiles, cli):
     project = str(datafiles)
diff --git a/tests/examples/autotools.py b/tests/examples/autotools.py
index ecd140718..efb8a67d3 100644
--- a/tests/examples/autotools.py
+++ b/tests/examples/autotools.py
@@ -16,10 +16,10 @@ DATA_DIR = os.path.join(
 
 
 # Tests a build of the autotools amhello project on a alpine-linux base runtime
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
+@pytest.mark.xfail(MACHINE_ARCH != 'x86-64',
                     reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
-@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
+@pytest.mark.xfail(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
 @pytest.mark.datafiles(DATA_DIR)
 def test_autotools_build(cli, datafiles):
     project = str(datafiles)
@@ -40,11 +40,11 @@ def test_autotools_build(cli, datafiles):
 
 
 # Test running an executable built with autotools.
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
+@pytest.mark.xfail(MACHINE_ARCH != 'x86-64',
                     reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
+@pytest.mark.xfail(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
 @pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
-@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
 @pytest.mark.datafiles(DATA_DIR)
 def test_autotools_run(cli, datafiles):
     project = str(datafiles)
diff --git a/tests/examples/developing.py b/tests/examples/developing.py
index 53a554b86..63052dcaa 100644
--- a/tests/examples/developing.py
+++ b/tests/examples/developing.py
@@ -17,10 +17,10 @@ DATA_DIR = os.path.join(
 
 
 # Test that the project builds successfully
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
+@pytest.mark.xfail(MACHINE_ARCH != 'x86-64',
                     reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with SANDBOX')
-@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This is not meant to work with chroot')
+@pytest.mark.xfail(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with SANDBOX')
+@pytest.mark.xfail(HAVE_SANDBOX == 'chroot', reason='This is not meant to work with chroot')
 @pytest.mark.datafiles(DATA_DIR)
 def test_autotools_build(cli, datafiles):
     project = str(datafiles)
@@ -39,10 +39,10 @@ def test_autotools_build(cli, datafiles):
 
 
 # Test the unmodified hello command works as expected.
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
+@pytest.mark.xfail(MACHINE_ARCH != 'x86-64',
                     reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with SANDBOX')
-@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This is not meant to work with chroot')
+@pytest.mark.xfail(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with SANDBOX')
+@pytest.mark.xfail(HAVE_SANDBOX == 'chroot', reason='This is not meant to work with chroot')
 @pytest.mark.datafiles(DATA_DIR)
 def test_run_unmodified_hello(cli, datafiles):
     project = str(datafiles)
@@ -56,7 +56,7 @@ def test_run_unmodified_hello(cli, datafiles):
 
 
 # Test opening a workspace
-@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
+@pytest.mark.xfail(not IS_LINUX, reason='Only available on linux')
 @pytest.mark.datafiles(DATA_DIR)
 def test_open_workspace(cli, tmpdir, datafiles):
     project = str(datafiles)
@@ -73,10 +73,10 @@ def test_open_workspace(cli, tmpdir, datafiles):
 
 
 # Test making a change using the workspace
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
+@pytest.mark.xfail(MACHINE_ARCH != 'x86-64',
                     reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with SANDBOX')
-@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This is not meant to work with chroot')
+@pytest.mark.xfail(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with SANDBOX')
+@pytest.mark.xfail(HAVE_SANDBOX == 'chroot', reason='This is not meant to work with chroot')
 @pytest.mark.datafiles(DATA_DIR)
 def test_make_change_in_workspace(cli, tmpdir, datafiles):
     project = str(datafiles)
diff --git a/tests/examples/first-project.py b/tests/examples/first-project.py
index 84ab7aa61..0c3b7e9a1 100644
--- a/tests/examples/first-project.py
+++ b/tests/examples/first-project.py
@@ -17,7 +17,7 @@ DATA_DIR = os.path.join(
 )
 
 
-@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
+@pytest.mark.xfail(not IS_LINUX, reason='Only available on linux')
 @pytest.mark.datafiles(DATA_DIR)
 def test_first_project_build_checkout(cli, datafiles):
     project = str(datafiles)
diff --git a/tests/examples/flatpak-autotools.py b/tests/examples/flatpak-autotools.py
index 2418807c0..337270d33 100644
--- a/tests/examples/flatpak-autotools.py
+++ b/tests/examples/flatpak-autotools.py
@@ -44,9 +44,9 @@ def workaround_setuptools_bug(project):
 
 # Test that a build upon flatpak runtime 'works' - we use the autotools sample
 # amhello project for this.
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
+@pytest.mark.xfail(MACHINE_ARCH != 'x86-64',
                     reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_OSTREE_PLUGIN, reason='Only available on linux with ostree')
+@pytest.mark.xfail(not IS_LINUX or not HAVE_OSTREE_PLUGIN, reason='Only available on linux with ostree')
 @pytest.mark.datafiles(DATA_DIR)
 def test_autotools_build(cli, datafiles):
     project = str(datafiles)
@@ -67,9 +67,9 @@ def test_autotools_build(cli, datafiles):
 
 
 # Test running an executable built with autotools
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
+@pytest.mark.xfail(MACHINE_ARCH != 'x86-64',
                     reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_OSTREE_PLUGIN, reason='Only available on linux with ostree')
+@pytest.mark.xfail(not IS_LINUX or not HAVE_OSTREE_PLUGIN, reason='Only available on linux with ostree')
 @pytest.mark.datafiles(DATA_DIR)
 def test_autotools_run(cli, datafiles):
     project = str(datafiles)
diff --git a/tests/examples/integration-commands.py b/tests/examples/integration-commands.py
index 1ed888b5d..b530c1423 100644
--- a/tests/examples/integration-commands.py
+++ b/tests/examples/integration-commands.py
@@ -14,10 +14,10 @@ DATA_DIR = os.path.join(
 )
 
 
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
+@pytest.mark.xfail(MACHINE_ARCH != 'x86-64',
                     reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
-@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
+@pytest.mark.xfail(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
 @pytest.mark.datafiles(DATA_DIR)
 def test_integration_commands_build(cli, datafiles):
     project = str(datafiles)
@@ -27,10 +27,10 @@ def test_integration_commands_build(cli, datafiles):
 
 
 # Test running the executable
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
+@pytest.mark.xfail(MACHINE_ARCH != 'x86-64',
                     reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
-@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
+@pytest.mark.xfail(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
 @pytest.mark.datafiles(DATA_DIR)
 def test_integration_commands_run(cli, datafiles):
     project = str(datafiles)
diff --git a/tests/examples/junctions.py b/tests/examples/junctions.py
index 18bf4da4f..d601b7450 100644
--- a/tests/examples/junctions.py
+++ b/tests/examples/junctions.py
@@ -15,10 +15,10 @@ DATA_DIR = os.path.join(
 
 
 # Test that the project builds successfully
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
+@pytest.mark.xfail(MACHINE_ARCH != 'x86-64',
                     reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with bubblewrap')
-@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
+@pytest.mark.xfail(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with bubblewrap')
+@pytest.mark.xfail(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
 @pytest.mark.datafiles(DATA_DIR)
 def test_build(cli, datafiles):
     project = str(datafiles)
@@ -28,10 +28,10 @@ def test_build(cli, datafiles):
 
 
 # Test the callHello script works as expected.
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
+@pytest.mark.xfail(MACHINE_ARCH != 'x86-64',
                     reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with bubblewrap')
-@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
+@pytest.mark.xfail(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with bubblewrap')
+@pytest.mark.xfail(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
 @pytest.mark.datafiles(DATA_DIR)
 def test_shell_call_hello(cli, datafiles):
     project = str(datafiles)
@@ -45,7 +45,7 @@ def test_shell_call_hello(cli, datafiles):
 
 
 # Test opening a cross-junction workspace
-@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
+@pytest.mark.xfail(not IS_LINUX, reason='Only available on linux')
 @pytest.mark.datafiles(DATA_DIR)
 def test_open_cross_junction_workspace(cli, tmpdir, datafiles):
     project = str(datafiles)
diff --git a/tests/examples/running-commands.py b/tests/examples/running-commands.py
index 1c419d524..f49d9cd59 100644
--- a/tests/examples/running-commands.py
+++ b/tests/examples/running-commands.py
@@ -14,11 +14,11 @@ DATA_DIR = os.path.join(
 )
 
 
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
+@pytest.mark.xfail(MACHINE_ARCH != 'x86-64',
                     reason='Examples are written for x86-64')
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
-@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
+@pytest.mark.xfail(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
 def test_running_commands_build(cli, datafiles):
     project = str(datafiles)
 
@@ -27,10 +27,10 @@ def test_running_commands_build(cli, datafiles):
 
 
 # Test running the executable
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
+@pytest.mark.xfail(MACHINE_ARCH != 'x86-64',
                     reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
-@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
+@pytest.mark.xfail(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
 @pytest.mark.datafiles(DATA_DIR)
 def test_running_commands_run(cli, datafiles):
     project = str(datafiles)
diff --git a/tests/format/junctions.py b/tests/format/junctions.py
index b810c55a3..91963c489 100644
--- a/tests/format/junctions.py
+++ b/tests/format/junctions.py
@@ -315,7 +315,7 @@ def test_options_inherit(cli, tmpdir, datafiles):
     assert os.path.exists(os.path.join(checkoutdir, 'horsy.txt'))
 
 
-@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.xfail(HAVE_GIT is False, reason="git is not available")
 @pytest.mark.datafiles(DATA_DIR)
 def test_git_show(cli, tmpdir, datafiles):
     project = os.path.join(str(datafiles), 'foo')
@@ -339,7 +339,7 @@ def test_git_show(cli, tmpdir, datafiles):
     assert 'base.bst:target.bst' in element_list
 
 
-@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.xfail(HAVE_GIT is False, reason="git is not available")
 @pytest.mark.datafiles(DATA_DIR)
 def test_git_build(cli, tmpdir, datafiles):
     project = os.path.join(str(datafiles), 'foo')
@@ -369,7 +369,7 @@ def test_git_build(cli, tmpdir, datafiles):
     assert os.path.exists(os.path.join(checkoutdir, 'foo.txt'))
 
 
-@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.xfail(HAVE_GIT is False, reason="git is not available")
 @pytest.mark.datafiles(DATA_DIR)
 def test_git_missing_project_conf(cli, tmpdir, datafiles):
     project = datafiles / 'foo'
diff --git a/tests/frontend/buildcheckout.py b/tests/frontend/buildcheckout.py
index 5079800ab..3e799c659 100644
--- a/tests/frontend/buildcheckout.py
+++ b/tests/frontend/buildcheckout.py
@@ -122,7 +122,7 @@ def test_build_invalid_suffix_dep(datafiles, cli, strict, hardlinks):
     result.assert_main_error(ErrorDomain.LOAD, "bad-element-suffix")
 
 
-@pytest.mark.skipif(IS_WINDOWS, reason='Not available on Windows')
+@pytest.mark.xfail(IS_WINDOWS, reason='Not available on Windows')
 @pytest.mark.datafiles(DATA_DIR)
 def test_build_invalid_filename_chars(datafiles, cli):
     project = str(datafiles)
@@ -139,7 +139,7 @@ def test_build_invalid_filename_chars(datafiles, cli):
     result.assert_main_error(ErrorDomain.LOAD, "bad-characters-in-name")
 
 
-@pytest.mark.skipif(IS_WINDOWS, reason='Not available on Windows')
+@pytest.mark.xfail(IS_WINDOWS, reason='Not available on Windows')
 @pytest.mark.datafiles(DATA_DIR)
 def test_build_invalid_filename_chars_dep(datafiles, cli):
     project = str(datafiles)
diff --git a/tests/frontend/track.py b/tests/frontend/track.py
index a628043d8..1e2fd3159 100644
--- a/tests/frontend/track.py
+++ b/tests/frontend/track.py
@@ -330,7 +330,7 @@ def test_junction_element(cli, tmpdir, datafiles, ref_storage):
 @pytest.mark.datafiles(DATA_DIR)
 def test_track_error_cannot_write_file(cli, tmpdir, datafiles):
     if os.geteuid() == 0:
-        pytest.skip("This is not testable with root permissions")
+        pytest.xfail("This is not testable with root permissions")
 
     project = str(datafiles)
     dev_files_path = os.path.join(project, 'files', 'dev-files')
diff --git a/tests/frontend/workspace.py b/tests/frontend/workspace.py
index 2006fe7f4..29e0e18e5 100644
--- a/tests/frontend/workspace.py
+++ b/tests/frontend/workspace.py
@@ -209,7 +209,7 @@ def test_open_multi(cli, tmpdir, datafiles):
             assert '.bzr' not in workspace_lsdir
 
 
-@pytest.mark.skipif(os.geteuid() == 0, reason="root may have CAP_DAC_OVERRIDE and ignore permissions")
+@pytest.mark.xfail(os.geteuid() == 0, reason="root may have CAP_DAC_OVERRIDE and ignore permissions")
 @pytest.mark.datafiles(DATA_DIR)
 def test_open_multi_unwritable(cli, tmpdir, datafiles):
     workspace_object = WorkspaceCreator(cli, tmpdir, datafiles)
diff --git a/tests/integration/artifact.py b/tests/integration/artifact.py
index 59b7bfaad..fe9e05e83 100644
--- a/tests/integration/artifact.py
+++ b/tests/integration/artifact.py
@@ -48,7 +48,7 @@ DATA_DIR = os.path.join(
 
 # Dse this really need a sandbox?
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_cache_buildtrees(cli, tmpdir, datafiles):
     project = str(datafiles)
     element_name = 'autotools/amhello.bst'
diff --git a/tests/integration/autotools.py b/tests/integration/autotools.py
index c985e8e6c..dbd335bbb 100644
--- a/tests/integration/autotools.py
+++ b/tests/integration/autotools.py
@@ -21,7 +21,7 @@ DATA_DIR = os.path.join(
 # Test that an autotools build 'works' - we use the autotools sample
 # amhello project for this.
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_autotools_build(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -43,7 +43,7 @@ def test_autotools_build(cli, datafiles):
 # Test that an autotools build 'works' - we use the autotools sample
 # amhello project for this.
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 @pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_autotools_confroot_build(cli, datafiles):
     project = str(datafiles)
@@ -65,7 +65,7 @@ def test_autotools_confroot_build(cli, datafiles):
 
 # Test running an executable built with autotools
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_autotools_run(cli, datafiles):
     project = str(datafiles)
     element_name = 'autotools/amhello.bst'
diff --git a/tests/integration/build-uid.py b/tests/integration/build-uid.py
index 66f9b3fbc..144065cac 100644
--- a/tests/integration/build-uid.py
+++ b/tests/integration/build-uid.py
@@ -16,7 +16,7 @@ DATA_DIR = os.path.join(
 )
 
 
-@pytest.mark.skipif(not IS_LINUX or HAVE_SANDBOX != "bwrap", reason='Only available on linux with bubblewrap')
+@pytest.mark.xfail(not IS_LINUX or HAVE_SANDBOX != "bwrap", reason='Only available on linux with bubblewrap')
 @pytest.mark.datafiles(DATA_DIR)
 def test_build_uid_overridden(cli, datafiles):
     project = str(datafiles)
@@ -35,7 +35,7 @@ def test_build_uid_overridden(cli, datafiles):
     assert result.exit_code == 0
 
 
-@pytest.mark.skipif(not IS_LINUX or HAVE_SANDBOX != "bwrap", reason='Only available on linux with bubbelwrap')
+@pytest.mark.xfail(not IS_LINUX or HAVE_SANDBOX != "bwrap", reason='Only available on linux with bubbelwrap')
 @pytest.mark.datafiles(DATA_DIR)
 def test_build_uid_in_project(cli, datafiles):
     project = str(datafiles)
@@ -55,7 +55,7 @@ def test_build_uid_in_project(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(HAVE_SANDBOX != "bwrap", reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX != "bwrap", reason='Only available with a functioning sandbox')
 def test_build_uid_default(cli, datafiles):
     project = str(datafiles)
     element_name = 'build-uid/build-uid-default.bst'
diff --git a/tests/integration/cachedfail.py b/tests/integration/cachedfail.py
index e3c05e691..8934258ab 100644
--- a/tests/integration/cachedfail.py
+++ b/tests/integration/cachedfail.py
@@ -38,7 +38,7 @@ DATA_DIR = os.path.join(
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_build_checkout_cached_fail(cli, datafiles):
     project = str(datafiles)
     element_path = os.path.join(project, 'elements', 'element.bst')
@@ -81,7 +81,7 @@ def test_build_checkout_cached_fail(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_build_depend_on_cached_fail(cli, datafiles):
     project = str(datafiles)
     dep_path = os.path.join(project, 'elements', 'dep.bst')
@@ -138,7 +138,7 @@ def test_build_depend_on_cached_fail(cli, datafiles):
     assert cli.get_element_state(project, 'target.bst') == 'waiting'
 
 
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 @pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.parametrize("on_error", ("continue", "quit"))
@@ -183,7 +183,7 @@ def test_push_cached_fail(cli, tmpdir, datafiles, on_error):
         assert share.get_artifact(cli.get_artifact_name(project, 'test', 'element.bst'))
 
 
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.parametrize("on_error", ("continue", "quit"))
 def test_push_failed_missing_shell(cli, tmpdir, datafiles, on_error):
@@ -226,7 +226,7 @@ def test_push_failed_missing_shell(cli, tmpdir, datafiles, on_error):
         assert share.get_artifact(cli.get_artifact_name(project, 'test', 'element.bst'))
 
 
-@pytest.mark.skipif(HAVE_SANDBOX != 'bwrap', reason='Only available with bubblewrap on Linux')
+@pytest.mark.xfail(HAVE_SANDBOX != 'bwrap', reason='Only available with bubblewrap on Linux')
 @pytest.mark.datafiles(DATA_DIR)
 def test_host_tools_errors_are_not_cached(cli, datafiles, tmp_path):
     # Create symlink to buildbox-casd to work with custom PATH
diff --git a/tests/integration/cmake.py b/tests/integration/cmake.py
index 0fe91ed6d..b1eb6a8f0 100644
--- a/tests/integration/cmake.py
+++ b/tests/integration/cmake.py
@@ -19,7 +19,7 @@ DATA_DIR = os.path.join(
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_cmake_build(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -35,7 +35,7 @@ def test_cmake_build(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 @pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_cmake_confroot_build(cli, datafiles):
     project = str(datafiles)
@@ -52,7 +52,7 @@ def test_cmake_confroot_build(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_cmake_run(cli, datafiles):
     project = str(datafiles)
     element_name = 'cmake/cmakehello.bst'
diff --git a/tests/integration/compose.py b/tests/integration/compose.py
index 3562ed94b..a88d5daf1 100644
--- a/tests/integration/compose.py
+++ b/tests/integration/compose.py
@@ -79,7 +79,7 @@ def create_compose_element(name, path, config=None):
                     '/usr/share/doc', '/usr/share/doc/amhello',
                     '/usr/share/doc/amhello/README'])
 ])
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_compose_include(cli, datafiles, include_domains,
                          exclude_domains, expected):
     project = str(datafiles)
diff --git a/tests/integration/filter.py b/tests/integration/filter.py
index 2a32d4010..817648d8b 100644
--- a/tests/integration/filter.py
+++ b/tests/integration/filter.py
@@ -20,7 +20,7 @@ DATA_DIR = os.path.join(
 
 
 @pytest.mark.datafiles(os.path.join(DATA_DIR))
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_filter_pass_integration(datafiles, cli):
     project = str(datafiles)
 
diff --git a/tests/integration/make.py b/tests/integration/make.py
index 664e7ca7a..014bb2e45 100644
--- a/tests/integration/make.py
+++ b/tests/integration/make.py
@@ -21,7 +21,7 @@ DATA_DIR = os.path.join(
 # Test that a make build 'works' - we use the make sample
 # makehello project for this.
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_make_build(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -39,7 +39,7 @@ def test_make_build(cli, datafiles):
 
 # Test running an executable built with make
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_make_run(cli, datafiles):
     project = str(datafiles)
     element_name = 'make/makehello.bst'
diff --git a/tests/integration/manual.py b/tests/integration/manual.py
index 2ac7f74d0..1b788ef79 100644
--- a/tests/integration/manual.py
+++ b/tests/integration/manual.py
@@ -35,7 +35,7 @@ def create_manual_element(name, path, config, variables, environment):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_manual_element(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -69,7 +69,7 @@ strip
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_manual_element_environment(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -99,7 +99,7 @@ def test_manual_element_environment(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_manual_element_noparallel(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -134,7 +134,7 @@ def test_manual_element_noparallel(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_manual_element_logging(cli, datafiles):
     project = str(datafiles)
     element_path = os.path.join(project, 'elements')
diff --git a/tests/integration/messages.py b/tests/integration/messages.py
index 42725fc5b..3706d9643 100644
--- a/tests/integration/messages.py
+++ b/tests/integration/messages.py
@@ -40,7 +40,7 @@ DATA_DIR = os.path.join(
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_disable_message_lines(cli, datafiles):
     project = str(datafiles)
     element_path = os.path.join(project, 'elements')
@@ -75,7 +75,7 @@ def test_disable_message_lines(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_disable_error_lines(cli, datafiles):
     project = str(datafiles)
     element_path = os.path.join(project, 'elements')
diff --git a/tests/integration/pip_element.py b/tests/integration/pip_element.py
index da0badcb3..1187ce04f 100644
--- a/tests/integration/pip_element.py
+++ b/tests/integration/pip_element.py
@@ -24,7 +24,7 @@ DATA_DIR = os.path.join(
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_pip_build(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -61,7 +61,7 @@ def test_pip_build(cli, datafiles):
 
 # Test running an executable built with pip
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_pip_run(cli, datafiles):
     # Create and build our test element
     test_pip_build(cli, datafiles)
@@ -75,7 +75,7 @@ def test_pip_run(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_pip_element_should_install_pip_deps(cli, datafiles, setup_pypi_repo):
     project = str(datafiles)
     elements_path = os.path.join(project, 'elements')
diff --git a/tests/integration/pip_source.py b/tests/integration/pip_source.py
index c221910a6..97c6048be 100644
--- a/tests/integration/pip_source.py
+++ b/tests/integration/pip_source.py
@@ -139,7 +139,7 @@ def test_pip_source_import_requirements_files(cli, datafiles, setup_pypi_repo):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_pip_source_build(cli, datafiles, setup_pypi_repo):
     project = str(datafiles)
     element_path = os.path.join(project, 'elements')
diff --git a/tests/integration/pullbuildtrees.py b/tests/integration/pullbuildtrees.py
index a1b188e5a..5af25ffc6 100644
--- a/tests/integration/pullbuildtrees.py
+++ b/tests/integration/pullbuildtrees.py
@@ -39,7 +39,7 @@ def default_state(cli, tmpdir, share):
 # directory of an element.
 @pytest.mark.integration
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_pullbuildtrees(cli2, tmpdir, datafiles):
     project = str(datafiles)
     element_name = 'autotools/amhello.bst'
diff --git a/tests/integration/sandbox-bwrap.py b/tests/integration/sandbox-bwrap.py
index f48c75cbd..ded790e2c 100644
--- a/tests/integration/sandbox-bwrap.py
+++ b/tests/integration/sandbox-bwrap.py
@@ -22,7 +22,7 @@ DATA_DIR = os.path.join(
 # Bubblewrap sandbox doesn't remove the dirs it created during its execution,
 # so BuildStream tries to remove them to do good. BuildStream should be extra
 # careful when those folders already exist and should not touch them, though.
-@pytest.mark.skipif(HAVE_SANDBOX != 'bwrap', reason='Only available with bubblewrap')
+@pytest.mark.xfail(HAVE_SANDBOX != 'bwrap', reason='Only available with bubblewrap')
 @pytest.mark.datafiles(DATA_DIR)
 def test_sandbox_bwrap_cleanup_build(cli, datafiles):
     project = str(datafiles)
@@ -34,8 +34,8 @@ def test_sandbox_bwrap_cleanup_build(cli, datafiles):
     assert result.exit_code == 0
 
 
-@pytest.mark.skipif(HAVE_SANDBOX != 'bwrap', reason='Only available with bubblewrap')
-@pytest.mark.skipif(not HAVE_BWRAP_JSON_STATUS, reason='Only available with bubblewrap supporting --json-status-fd')
+@pytest.mark.xfail(HAVE_SANDBOX != 'bwrap', reason='Only available with bubblewrap')
+@pytest.mark.xfail(not HAVE_BWRAP_JSON_STATUS, reason='Only available with bubblewrap supporting --json-status-fd')
 @pytest.mark.datafiles(DATA_DIR)
 def test_sandbox_bwrap_distinguish_setup_error(cli, datafiles):
     project = str(datafiles)
@@ -45,7 +45,7 @@ def test_sandbox_bwrap_distinguish_setup_error(cli, datafiles):
     result.assert_task_error(error_domain=ErrorDomain.SANDBOX, error_reason="bwrap-sandbox-fail")
 
 
-@pytest.mark.skipif(HAVE_SANDBOX != 'bwrap', reason='Only available with bubblewrap')
+@pytest.mark.xfail(HAVE_SANDBOX != 'bwrap', reason='Only available with bubblewrap')
 @pytest.mark.datafiles(DATA_DIR)
 def test_sandbox_bwrap_return_subprocess(cli, datafiles):
     project = str(datafiles)
diff --git a/tests/integration/script.py b/tests/integration/script.py
index dc893f679..31d1d5800 100644
--- a/tests/integration/script.py
+++ b/tests/integration/script.py
@@ -39,7 +39,7 @@ def create_script_element(name, path, config=None, variables=None):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_script(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -67,7 +67,7 @@ def test_script(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_script_root(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -99,7 +99,7 @@ def test_script_root(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 @pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_script_no_root(cli, datafiles):
     project = str(datafiles)
@@ -123,7 +123,7 @@ def test_script_no_root(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 @pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_script_cwd(cli, datafiles):
     project = str(datafiles)
@@ -155,7 +155,7 @@ def test_script_cwd(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_script_layout(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -174,7 +174,7 @@ def test_script_layout(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_regression_cache_corruption(cli, datafiles):
     project = str(datafiles)
     checkout_original = os.path.join(cli.directory, 'checkout-original')
@@ -204,7 +204,7 @@ def test_regression_cache_corruption(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_regression_tmpdir(cli, datafiles):
     project = str(datafiles)
     element_name = 'script/tmpdir.bst'
@@ -214,7 +214,7 @@ def test_regression_tmpdir(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_regression_cache_corruption_2(cli, datafiles):
     project = str(datafiles)
     checkout_original = os.path.join(cli.directory, 'checkout-original')
diff --git a/tests/integration/shell.py b/tests/integration/shell.py
index b0de76ef5..a82329c42 100644
--- a/tests/integration/shell.py
+++ b/tests/integration/shell.py
@@ -56,7 +56,7 @@ def execute_shell(cli, project, command, *, config=None, mount=None, element='ba
 # Test running something through a shell, allowing it to find the
 # executable
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_shell(cli, datafiles):
     project = str(datafiles)
 
@@ -67,7 +67,7 @@ def test_shell(cli, datafiles):
 
 # Test running an executable directly
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_executable(cli, datafiles):
     project = str(datafiles)
 
@@ -79,7 +79,7 @@ def test_executable(cli, datafiles):
 # Test shell environment variable explicit assignments
 @pytest.mark.parametrize("animal", [("Horse"), ("Pony")])
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 # This test seems to fail or pass depending on if this file is run or the hole test suite
 def test_env_assign(cli, datafiles, animal):
     project = str(datafiles)
@@ -100,7 +100,7 @@ def test_env_assign(cli, datafiles, animal):
 # Test shell environment variable explicit assignments with host env var expansion
 @pytest.mark.parametrize("animal", [("Horse"), ("Pony")])
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 # This test seems to fail or pass depending on if this file is run or the hole test suite
 def test_env_assign_expand_host_environ(cli, datafiles, animal):
     project = str(datafiles)
@@ -124,7 +124,7 @@ def test_env_assign_expand_host_environ(cli, datafiles, animal):
 # when running an isolated shell
 @pytest.mark.parametrize("animal", [("Horse"), ("Pony")])
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 # This test seems to faili or pass depending on if this file is run or the hole test suite
 def test_env_assign_isolated(cli, datafiles, animal):
     project = str(datafiles)
@@ -143,7 +143,7 @@ def test_env_assign_isolated(cli, datafiles, animal):
 # Test running an executable in a runtime with no shell (i.e., no
 # /bin/sh)
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_no_shell(cli, datafiles):
     project = str(datafiles)
     element_path = os.path.join(project, 'elements')
@@ -176,7 +176,7 @@ def test_no_shell(cli, datafiles):
 # Test that bind mounts defined in project.conf work
 @pytest.mark.parametrize("path", [("/etc/pony.conf"), ("/usr/share/pony/pony.txt")])
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 @pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_host_files(cli, datafiles, path):
     project = str(datafiles)
@@ -198,7 +198,7 @@ def test_host_files(cli, datafiles, path):
 # Test that bind mounts defined in project.conf work
 @pytest.mark.parametrize("path", [("/etc"), ("/usr/share/pony")])
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 @pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_host_files_expand_environ(cli, datafiles, path):
     project = str(datafiles)
@@ -225,7 +225,7 @@ def test_host_files_expand_environ(cli, datafiles, path):
 # Test that bind mounts defined in project.conf dont mount in isolation
 @pytest.mark.parametrize("path", [("/etc/pony.conf"), ("/usr/share/pony/pony.txt")])
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_isolated_no_mount(cli, datafiles, path):
     project = str(datafiles)
     ponyfile = os.path.join(project, 'files', 'shell-mount', 'pony.txt')
@@ -248,7 +248,7 @@ def test_isolated_no_mount(cli, datafiles, path):
 # declared as optional, and that there is no warning if it is optional
 @pytest.mark.parametrize("optional", [("mandatory"), ("optional")])
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_host_files_missing(cli, datafiles, optional):
     project = str(datafiles)
     ponyfile = os.path.join(project, 'files', 'shell-mount', 'horsy.txt')
@@ -281,7 +281,7 @@ def test_host_files_missing(cli, datafiles, optional):
 # Test that bind mounts defined in project.conf work
 @pytest.mark.parametrize("path", [("/etc/pony.conf"), ("/usr/share/pony/pony.txt")])
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 @pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_cli_mount(cli, datafiles, path):
     project = str(datafiles)
@@ -294,7 +294,7 @@ def test_cli_mount(cli, datafiles, path):
 
 # Test that we can see the workspace files in a shell
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_workspace_visible(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -328,7 +328,7 @@ def test_workspace_visible(cli, datafiles):
 
 # Test that '--sysroot' works
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 @pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_sysroot(cli, tmpdir, datafiles):
     project = str(datafiles)
@@ -359,7 +359,7 @@ def test_sysroot(cli, tmpdir, datafiles):
 
 # Test system integration commands can access devices in /dev
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_integration_devices(cli, datafiles):
     project = str(datafiles)
     element_name = 'integration.bst'
@@ -372,7 +372,7 @@ def test_integration_devices(cli, datafiles):
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.parametrize("build_shell", [("build"), ("nobuild")])
 @pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"])
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_integration_external_workspace(cli, tmpdir_factory, datafiles, build_shell, guess_element):
     tmpdir = tmpdir_factory.mktemp("")
     project = str(datafiles)
@@ -405,7 +405,7 @@ def test_integration_external_workspace(cli, tmpdir_factory, datafiles, build_sh
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_integration_partial_artifact(cli, datafiles, tmpdir, integration_cache):
 
     project = str(datafiles)
diff --git a/tests/integration/shellbuildtrees.py b/tests/integration/shellbuildtrees.py
index 146bc6062..1c2593109 100644
--- a/tests/integration/shellbuildtrees.py
+++ b/tests/integration/shellbuildtrees.py
@@ -23,7 +23,7 @@ DATA_DIR = os.path.join(
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_buildtree_staged(cli_integration, datafiles):
     # We can only test the non interacitve case
     # The non interactive case defaults to not using buildtrees
@@ -41,7 +41,7 @@ def test_buildtree_staged(cli_integration, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_buildtree_staged_forced_true(cli_integration, datafiles):
     # Test that if we ask for a build tree it is there.
     project = str(datafiles)
@@ -58,7 +58,7 @@ def test_buildtree_staged_forced_true(cli_integration, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_buildtree_staged_warn_empty_cached(cli_integration, tmpdir, datafiles):
     # Test that if we stage a cached and empty buildtree, we warn the user.
     project = str(datafiles)
@@ -89,7 +89,7 @@ def test_buildtree_staged_warn_empty_cached(cli_integration, tmpdir, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_buildtree_staged_if_available(cli_integration, datafiles):
     # Test that a build tree can be correctly detected.
     project = str(datafiles)
@@ -106,7 +106,7 @@ def test_buildtree_staged_if_available(cli_integration, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_buildtree_staged_forced_false(cli_integration, datafiles):
     # Test that if we ask not to have a build tree it is not there
     project = str(datafiles)
@@ -124,7 +124,7 @@ def test_buildtree_staged_forced_false(cli_integration, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_buildtree_from_failure(cli_integration, datafiles):
     # Test that we can use a build tree after a failure
     project = str(datafiles)
@@ -143,7 +143,7 @@ def test_buildtree_from_failure(cli_integration, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_buildtree_from_failure_option_never(cli_integration, tmpdir, datafiles):
 
     project = str(datafiles)
@@ -166,7 +166,7 @@ def test_buildtree_from_failure_option_never(cli_integration, tmpdir, datafiles)
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_buildtree_from_failure_option_always(cli_integration, tmpdir, datafiles):
 
     project = str(datafiles)
@@ -193,7 +193,7 @@ def test_buildtree_from_failure_option_always(cli_integration, tmpdir, datafiles
 # Check that build shells work when pulled from a remote cache
 # This is to roughly simulate remote execution
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_buildtree_pulled(cli, tmpdir, datafiles):
     project = str(datafiles)
     element_name = 'build-shell/buildtree.bst'
@@ -226,7 +226,7 @@ def test_buildtree_pulled(cli, tmpdir, datafiles):
 
 # This test checks for correct behaviour if a buildtree is not present in the local cache.
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_buildtree_options(cli, tmpdir, datafiles):
     project = str(datafiles)
     element_name = 'build-shell/buildtree.bst'
@@ -309,7 +309,7 @@ def test_buildtree_options(cli, tmpdir, datafiles):
 
 # Tests running pull and pull-buildtree options at the same time.
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_pull_buildtree_pulled(cli, tmpdir, datafiles):
     project = str(datafiles)
     element_name = 'build-shell/buildtree.bst'
diff --git a/tests/integration/sockets.py b/tests/integration/sockets.py
index 763238baf..7c5a9957a 100644
--- a/tests/integration/sockets.py
+++ b/tests/integration/sockets.py
@@ -17,7 +17,7 @@ DATA_DIR = os.path.join(
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_builddir_socket_ignored(cli, datafiles):
     project = str(datafiles)
     element_name = 'sockets/make-builddir-socket.bst'
@@ -27,7 +27,7 @@ def test_builddir_socket_ignored(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_install_root_socket_ignored(cli, datafiles):
     project = str(datafiles)
     element_name = 'sockets/make-install-root-socket.bst'
diff --git a/tests/integration/source-determinism.py b/tests/integration/source-determinism.py
index 70c4b79de..fce63cee2 100644
--- a/tests/integration/source-determinism.py
+++ b/tests/integration/source-determinism.py
@@ -31,7 +31,7 @@ def create_test_directory(*path, mode=0o644):
 
 @pytest.mark.integration
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_deterministic_source_local(cli, tmpdir, datafiles):
     """Only user rights should be considered for local source.
     """
diff --git a/tests/integration/stack.py b/tests/integration/stack.py
index 9d6b38345..5dcc931dc 100644
--- a/tests/integration/stack.py
+++ b/tests/integration/stack.py
@@ -18,7 +18,7 @@ DATA_DIR = os.path.join(
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_stack(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
diff --git a/tests/integration/symlinks.py b/tests/integration/symlinks.py
index 85bbc53fd..6fc92cf0f 100644
--- a/tests/integration/symlinks.py
+++ b/tests/integration/symlinks.py
@@ -18,7 +18,7 @@ DATA_DIR = os.path.join(
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_absolute_symlinks(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -39,7 +39,7 @@ def test_absolute_symlinks(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_disallow_overlaps_inside_symlink_with_dangling_target(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -54,7 +54,7 @@ def test_disallow_overlaps_inside_symlink_with_dangling_target(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_detect_symlink_overlaps_pointing_outside_sandbox(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -73,7 +73,7 @@ def test_detect_symlink_overlaps_pointing_outside_sandbox(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_symlink_in_sandbox_path(cli, datafiles):
     project = str(datafiles)
     element_name = 'symlinks/link-on-path-use.bst'
diff --git a/tests/integration/workspace.py b/tests/integration/workspace.py
index a4cb454d1..4a2cbc68d 100644
--- a/tests/integration/workspace.py
+++ b/tests/integration/workspace.py
@@ -20,7 +20,7 @@ DATA_DIR = os.path.join(
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_workspace_mount(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -50,7 +50,7 @@ def test_workspace_mount_on_read_only_directory(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 @pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 @pytest.mark.xfail(reason="Incremental builds are currently incompatible with workspace source plugin.", strict=True)
 def test_workspace_commanddir(cli, datafiles):
@@ -69,7 +69,7 @@ def test_workspace_commanddir(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_workspace_updated_dependency(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -123,7 +123,7 @@ def test_workspace_updated_dependency(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_workspace_update_dependency_failed(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -198,7 +198,7 @@ def test_workspace_update_dependency_failed(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_updated_dependency_nested(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -251,7 +251,7 @@ def test_updated_dependency_nested(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 @pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 @pytest.mark.xfail(reason="Incremental builds are currently incompatible with workspace source plugin.", strict=True)
 def test_incremental_configure_commands_run_only_once(cli, datafiles):
@@ -305,7 +305,7 @@ def test_incremental_configure_commands_run_only_once(cli, datafiles):
 # part of a cleanup job.
 #
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 @pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_workspace_missing_last_successful(cli, datafiles):
     project = str(datafiles)
@@ -331,7 +331,7 @@ def test_workspace_missing_last_successful(cli, datafiles):
 
 # Check that we can still read failed workspace logs
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_workspace_failed_logs(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'failing_amhello')
diff --git a/tests/sandboxes/missing_dependencies.py b/tests/sandboxes/missing_dependencies.py
index 975c8eb00..f83d92474 100644
--- a/tests/sandboxes/missing_dependencies.py
+++ b/tests/sandboxes/missing_dependencies.py
@@ -18,7 +18,7 @@ DATA_DIR = os.path.join(
 )
 
 
-@pytest.mark.skipif(not IS_LINUX, reason='Only available on Linux')
+@pytest.mark.xfail(not IS_LINUX, reason='Only available on Linux')
 @pytest.mark.datafiles(DATA_DIR)
 def test_missing_brwap_has_nice_error_message(cli, datafiles, tmp_path):
     # Create symlink to buildbox-casd to work with custom PATH
@@ -56,7 +56,7 @@ def test_missing_brwap_has_nice_error_message(cli, datafiles, tmp_path):
     assert "not found" in result.stderr
 
 
-@pytest.mark.skipif(not IS_LINUX, reason='Only available on Linux')
+@pytest.mark.xfail(not IS_LINUX, reason='Only available on Linux')
 @pytest.mark.datafiles(DATA_DIR)
 def test_old_brwap_has_nice_error_message(cli, datafiles, tmp_path):
     bwrap = tmp_path.joinpath('bin/bwrap')
diff --git a/tests/sandboxes/mounting/mount_simple.py b/tests/sandboxes/mounting/mount_simple.py
index 65aaf209d..5d88637c8 100644
--- a/tests/sandboxes/mounting/mount_simple.py
+++ b/tests/sandboxes/mounting/mount_simple.py
@@ -7,7 +7,7 @@ import pytest
 from buildstream.sandbox._mounter import Mounter
 
 
-@pytest.mark.skipif(not os.geteuid() == 0, reason="requires root permissions")
+@pytest.mark.xfail(not os.geteuid() == 0, reason="requires root permissions")
 def test_bind_mount():
     with ExitStack() as stack:
         src = stack.enter_context(tempfile.TemporaryDirectory())
@@ -35,7 +35,7 @@ def test_bind_mount():
             assert test.read() == 'Test'
 
 
-@pytest.mark.skipif(not os.geteuid() == 0, reason="requires root permissions")
+@pytest.mark.xfail(not os.geteuid() == 0, reason="requires root permissions")
 def test_mount_proc():
     with ExitStack() as stack:
         src = '/proc'
diff --git a/tests/sources/bzr.py b/tests/sources/bzr.py
index c6e78f8c1..1d33c4301 100644
--- a/tests/sources/bzr.py
+++ b/tests/sources/bzr.py
@@ -16,7 +16,7 @@ DATA_DIR = os.path.join(
 )
 
 
-@pytest.mark.skipif(HAVE_BZR is False, reason="bzr is not available")
+@pytest.mark.xfail(HAVE_BZR is False, reason="bzr is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR))
 def test_fetch_checkout(cli, tmpdir, datafiles):
     project = str(datafiles)
diff --git a/tests/sources/deb.py b/tests/sources/deb.py
index e536e522a..5e2af73e1 100644
--- a/tests/sources/deb.py
+++ b/tests/sources/deb.py
@@ -37,7 +37,7 @@ def _copy_deb(start_location, tmpdir):
 
 
 # Test that without ref, consistency is set appropriately.
-@pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available")
+@pytest.mark.xfail(HAVE_ARPY is False, reason="arpy is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref'))
 def test_no_ref(cli, tmpdir, datafiles):
     project = str(datafiles)
@@ -46,7 +46,7 @@ def test_no_ref(cli, tmpdir, datafiles):
 
 
 # Test that when I fetch a nonexistent URL, errors are handled gracefully and a retry is performed.
-@pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available")
+@pytest.mark.xfail(HAVE_ARPY is False, reason="arpy is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
 def test_fetch_bad_url(cli, tmpdir, datafiles):
     project = str(datafiles)
@@ -61,7 +61,7 @@ def test_fetch_bad_url(cli, tmpdir, datafiles):
     result.assert_task_error(ErrorDomain.SOURCE, None)
 
 
-@pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available")
+@pytest.mark.xfail(HAVE_ARPY is False, reason="arpy is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
 def test_fetch_bad_ref(cli, tmpdir, datafiles):
     project = str(datafiles)
@@ -79,7 +79,7 @@ def test_fetch_bad_ref(cli, tmpdir, datafiles):
 
 
 # Test that when tracking with a ref set, there is a warning
-@pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available")
+@pytest.mark.xfail(HAVE_ARPY is False, reason="arpy is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
 def test_track_warning(cli, tmpdir, datafiles):
     project = str(datafiles)
@@ -97,7 +97,7 @@ def test_track_warning(cli, tmpdir, datafiles):
 
 
 # Test that a staged checkout matches what was tarred up, with the default first subdir
-@pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available")
+@pytest.mark.xfail(HAVE_ARPY is False, reason="arpy is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
 def test_stage_default_basedir(cli, tmpdir, datafiles):
     project = str(datafiles)
@@ -125,7 +125,7 @@ def test_stage_default_basedir(cli, tmpdir, datafiles):
 
 
 # Test that a staged checkout matches what was tarred up, with an empty base-dir
-@pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available")
+@pytest.mark.xfail(HAVE_ARPY is False, reason="arpy is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-basedir'))
 def test_stage_no_basedir(cli, tmpdir, datafiles):
     project = str(datafiles)
@@ -153,7 +153,7 @@ def test_stage_no_basedir(cli, tmpdir, datafiles):
 
 
 # Test that a staged checkout matches what was tarred up, with an explicit basedir
-@pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available")
+@pytest.mark.xfail(HAVE_ARPY is False, reason="arpy is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'explicit-basedir'))
 def test_stage_explicit_basedir(cli, tmpdir, datafiles):
     project = str(datafiles)
diff --git a/tests/sources/git.py b/tests/sources/git.py
index 245f90131..604548a13 100644
--- a/tests/sources/git.py
+++ b/tests/sources/git.py
@@ -42,7 +42,7 @@ DATA_DIR = os.path.join(
 )
 
 
-@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.xfail(HAVE_GIT is False, reason="git is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
 def test_fetch_bad_ref(cli, tmpdir, datafiles):
     project = str(datafiles)
@@ -68,7 +68,7 @@ def test_fetch_bad_ref(cli, tmpdir, datafiles):
     result.assert_task_error(ErrorDomain.SOURCE, None)
 
 
-@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.xfail(HAVE_GIT is False, reason="git is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
 def test_submodule_fetch_checkout(cli, tmpdir, datafiles):
     project = str(datafiles)
@@ -107,7 +107,7 @@ def test_submodule_fetch_checkout(cli, tmpdir, datafiles):
     assert os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt'))
 
 
-@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.xfail(HAVE_GIT is False, reason="git is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
 def test_submodule_fetch_source_enable_explicit(cli, tmpdir, datafiles):
     project = str(datafiles)
@@ -146,7 +146,7 @@ def test_submodule_fetch_source_enable_explicit(cli, tmpdir, datafiles):
     assert os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt'))
 
 
-@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.xfail(HAVE_GIT is False, reason="git is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
 def test_submodule_fetch_source_disable(cli, tmpdir, datafiles):
     project = str(datafiles)
@@ -185,7 +185,7 @@ def test_submodule_fetch_source_disable(cli, tmpdir, datafiles):
     assert not os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt'))
 
 
-@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.xfail(HAVE_GIT is False, reason="git is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
 def test_submodule_fetch_submodule_does_override(cli, tmpdir, datafiles):
     project = str(datafiles)
@@ -224,7 +224,7 @@ def test_submodule_fetch_submodule_does_override(cli, tmpdir, datafiles):
     assert os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt'))
 
 
-@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.xfail(HAVE_GIT is False, reason="git is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
 def test_submodule_fetch_submodule_individual_checkout(cli, tmpdir, datafiles):
     project = str(datafiles)
@@ -269,7 +269,7 @@ def test_submodule_fetch_submodule_individual_checkout(cli, tmpdir, datafiles):
     assert os.path.exists(os.path.join(checkoutdir, 'othersubdir', 'unicornfile.txt'))
 
 
-@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.xfail(HAVE_GIT is False, reason="git is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
 def test_submodule_fetch_submodule_individual_checkout_explicit(cli, tmpdir, datafiles):
     project = str(datafiles)
@@ -314,7 +314,7 @@ def test_submodule_fetch_submodule_individual_checkout_explicit(cli, tmpdir, dat
     assert os.path.exists(os.path.join(checkoutdir, 'othersubdir', 'unicornfile.txt'))
 
 
-@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.xfail(HAVE_GIT is False, reason="git is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'project-override'))
 def test_submodule_fetch_project_override(cli, tmpdir, datafiles):
     project = str(datafiles)
@@ -353,7 +353,7 @@ def test_submodule_fetch_project_override(cli, tmpdir, datafiles):
     assert not os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt'))
 
 
-@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.xfail(HAVE_GIT is False, reason="git is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
 def test_submodule_track_ignore_inconsistent(cli, tmpdir, datafiles):
     project = str(datafiles)
@@ -390,7 +390,7 @@ def test_submodule_track_ignore_inconsistent(cli, tmpdir, datafiles):
     assert "Ignoring inconsistent submodule" in result.stderr
 
 
-@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.xfail(HAVE_GIT is False, reason="git is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
 def test_submodule_track_no_ref_or_track(cli, tmpdir, datafiles):
     project = str(datafiles)
@@ -417,7 +417,7 @@ def test_submodule_track_no_ref_or_track(cli, tmpdir, datafiles):
     result.assert_task_error(None, None)
 
 
-@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.xfail(HAVE_GIT is False, reason="git is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
 @pytest.mark.parametrize("fail", ['warn', 'error'])
 def test_ref_not_in_track(cli, tmpdir, datafiles, fail):
@@ -460,7 +460,7 @@ def test_ref_not_in_track(cli, tmpdir, datafiles, fail):
         assert "ref-not-in-track" in result.stderr
 
 
-@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.xfail(HAVE_GIT is False, reason="git is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
 @pytest.mark.parametrize("fail", ['warn', 'error'])
 def test_unlisted_submodule(cli, tmpdir, datafiles, fail):
@@ -533,7 +533,7 @@ def test_unlisted_submodule(cli, tmpdir, datafiles, fail):
         assert "git:unlisted-submodule" in result.stderr
 
 
-@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.xfail(HAVE_GIT is False, reason="git is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
 @pytest.mark.parametrize("fail", ['warn', 'error'])
 def test_track_unlisted_submodule(cli, tmpdir, datafiles, fail):
@@ -596,7 +596,7 @@ def test_track_unlisted_submodule(cli, tmpdir, datafiles, fail):
         assert "git:unlisted-submodule" in result.stderr
 
 
-@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.xfail(HAVE_GIT is False, reason="git is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
 @pytest.mark.parametrize("fail", ['warn', 'error'])
 def test_invalid_submodule(cli, tmpdir, datafiles, fail):
@@ -667,8 +667,8 @@ def test_invalid_submodule(cli, tmpdir, datafiles, fail):
         assert "git:invalid-submodule" in result.stderr
 
 
-@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
-@pytest.mark.skipif(HAVE_OLD_GIT, reason="old git rm does not update .gitmodules")
+@pytest.mark.xfail(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.xfail(HAVE_OLD_GIT, reason="old git rm does not update .gitmodules")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
 @pytest.mark.parametrize("fail", ['warn', 'error'])
 def test_track_invalid_submodule(cli, tmpdir, datafiles, fail):
@@ -729,7 +729,7 @@ def test_track_invalid_submodule(cli, tmpdir, datafiles, fail):
         assert "git:invalid-submodule" in result.stderr
 
 
-@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.xfail(HAVE_GIT is False, reason="git is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
 @pytest.mark.parametrize("ref_format", ['sha1', 'git-describe'])
 @pytest.mark.parametrize("tag,extra_commit", [(False, False), (True, False), (True, True)])
@@ -776,8 +776,8 @@ def test_track_fetch(cli, tmpdir, datafiles, ref_format, tag, extra_commit):
     result.assert_success()
 
 
-@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
-@pytest.mark.skipif(HAVE_OLD_GIT, reason="old git describe lacks --first-parent")
+@pytest.mark.xfail(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.xfail(HAVE_OLD_GIT, reason="old git describe lacks --first-parent")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
 @pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
 @pytest.mark.parametrize("tag_type", [('annotated'), ('lightweight')])
@@ -891,7 +891,7 @@ def test_git_describe(cli, tmpdir, datafiles, ref_storage, tag_type):
     assert p.returncode != 0
 
 
-@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.xfail(HAVE_GIT is False, reason="git is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
 @pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
 @pytest.mark.parametrize("tag_type", [('annotated'), ('lightweight')])
@@ -1008,7 +1008,7 @@ def test_git_describe_head_is_tagged(cli, tmpdir, datafiles, ref_storage, tag_ty
     assert p.returncode != 0
 
 
-@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.xfail(HAVE_GIT is False, reason="git is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
 def test_git_describe_relevant_history(cli, tmpdir, datafiles):
     project = str(datafiles)
@@ -1088,7 +1088,7 @@ def test_git_describe_relevant_history(cli, tmpdir, datafiles):
     assert set(rev_list.splitlines()) == set([head, tagged_ref, branch_boundary])
 
 
-@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.xfail(HAVE_GIT is False, reason="git is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
 def test_default_do_not_track_tags(cli, tmpdir, datafiles):
     project = str(datafiles)
@@ -1129,7 +1129,7 @@ def test_default_do_not_track_tags(cli, tmpdir, datafiles):
     assert 'tags' not in source
 
 
-@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.xfail(HAVE_GIT is False, reason="git is not available")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
 def test_overwrite_rogue_tag_multiple_remotes(cli, tmpdir, datafiles):
     """When using multiple remotes in cache (i.e. when using aliases), we
diff --git a/tests/sources/local.py b/tests/sources/local.py
index 4b72a4343..7c54b7f8f 100644
--- a/tests/sources/local.py
+++ b/tests/sources/local.py
@@ -164,7 +164,7 @@ def test_stage_directory_symlink(cli, tmpdir, datafiles):
 
 @pytest.mark.integration
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'deterministic-umask'))
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_deterministic_source_umask(cli, tmpdir, datafiles):
 
     def create_test_file(*path, mode=0o644, content='content\n'):
diff --git a/tests/sources/no_fetch_cached.py b/tests/sources/no_fetch_cached.py
index 81032881c..f7b782424 100644
--- a/tests/sources/no_fetch_cached.py
+++ b/tests/sources/no_fetch_cached.py
@@ -20,7 +20,7 @@ DATA_DIR = os.path.join(
 #                              Tests                             #
 ##################################################################
 # Test that fetch() is not called for cached sources
-@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.xfail(HAVE_GIT is False, reason="git is not available")
 @pytest.mark.datafiles(DATA_DIR)
 def test_no_fetch_cached(cli, tmpdir, datafiles):
     project = str(datafiles)
diff --git a/tests/sources/tar.py b/tests/sources/tar.py
index fac6f3f8b..315d58a74 100644
--- a/tests/sources/tar.py
+++ b/tests/sources/tar.py
@@ -247,7 +247,7 @@ def test_stage_contains_links(cli, tmpdir, datafiles):
     assert checkout_contents == original_contents
 
 
-@pytest.mark.skipif(not HAVE_LZIP, reason='lzip is not available')
+@pytest.mark.xfail(not HAVE_LZIP, reason='lzip is not available')
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
 @pytest.mark.parametrize("srcdir", ["a", "./a"])
 def test_stage_default_basedir_lzip(cli, tmpdir, datafiles, srcdir):
diff --git a/tests/testutils/repo/bzr.py b/tests/testutils/repo/bzr.py
index 074712af1..a62e9d9e0 100644
--- a/tests/testutils/repo/bzr.py
+++ b/tests/testutils/repo/bzr.py
@@ -10,7 +10,7 @@ class Bzr(Repo):
 
     def __init__(self, directory, subdir):
         if not HAVE_BZR:
-            pytest.skip("bzr is not available")
+            pytest.xfail("bzr is not available")
         super().__init__(directory, subdir)
         self.bzr = BZR
 
diff --git a/tests/testutils/repo/git.py b/tests/testutils/repo/git.py
index 46694fcf2..7857d6145 100644
--- a/tests/testutils/repo/git.py
+++ b/tests/testutils/repo/git.py
@@ -12,7 +12,7 @@ class Git(Repo):
 
     def __init__(self, directory, subdir):
         if not HAVE_GIT:
-            pytest.skip("git is not available")
+            pytest.xfail("git is not available")
 
         self.submodules = {}
author	Benjamin Schubert <contact@benschubert.me>	2019-10-09 10:56:29 +0100
committer	Benjamin Schubert <contact@benschubert.me>	2019-10-09 10:56:29 +0100
commit	b1b0d28e5051a78cd7b04fe70ed9dffb1f62069f (patch)
tree	3fcb14c9b33ead2cd5913498b0dfce58247ae78d
parent	b592cb85c7e8f3580e75dff531d44ec1bed84dc1 (diff)
download	buildstream-bschubert/test-skips.tar.gz