Merge branch 'juerg/buildbox' into 'master'

Enable integration tests for buildbox sandboxing backend

See merge request BuildStream/buildstream!1525

24 files changed, 3 insertions, 123 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bf0b8f557..c18872083 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -111,10 +111,7 @@ tests-buildbox:
 
   script:
 
-    # We remove the Bubblewrap package here so that we catch any
-    # codepaths that try to use them.
     - dnf install -y fuse3
-    - dnf erase -y bubblewrap
 
     # Before buildbox is a first class citizen we need a good install story for users and this test
     # should mirror that story, for now we build in the test as it is quick and easy.
diff --git a/tests/examples/autotools.py b/tests/examples/autotools.py
index e55f33493..ecd140718 100644
--- a/tests/examples/autotools.py
+++ b/tests/examples/autotools.py
@@ -20,8 +20,6 @@ DATA_DIR = os.path.join(
                     reason='Examples are written for x86-64')
 @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
 @pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 @pytest.mark.datafiles(DATA_DIR)
 def test_autotools_build(cli, datafiles):
     project = str(datafiles)
diff --git a/tests/examples/developing.py b/tests/examples/developing.py
index 58416414e..2f53c8a33 100644
--- a/tests/examples/developing.py
+++ b/tests/examples/developing.py
@@ -21,8 +21,6 @@ DATA_DIR = os.path.join(
                     reason='Examples are written for x86-64')
 @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with SANDBOX')
 @pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This is not meant to work with chroot')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 @pytest.mark.datafiles(DATA_DIR)
 def test_autotools_build(cli, datafiles):
     project = str(datafiles)
@@ -45,8 +43,6 @@ def test_autotools_build(cli, datafiles):
                     reason='Examples are written for x86-64')
 @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with SANDBOX')
 @pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This is not meant to work with chroot')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 @pytest.mark.datafiles(DATA_DIR)
 def test_run_unmodified_hello(cli, datafiles):
     project = str(datafiles)
diff --git a/tests/examples/integration-commands.py b/tests/examples/integration-commands.py
index 146ccdb80..1ed888b5d 100644
--- a/tests/examples/integration-commands.py
+++ b/tests/examples/integration-commands.py
@@ -18,8 +18,6 @@ DATA_DIR = os.path.join(
                     reason='Examples are written for x86-64')
 @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
 @pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 @pytest.mark.datafiles(DATA_DIR)
 def test_integration_commands_build(cli, datafiles):
     project = str(datafiles)
@@ -33,8 +31,6 @@ def test_integration_commands_build(cli, datafiles):
                     reason='Examples are written for x86-64')
 @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
 @pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 @pytest.mark.datafiles(DATA_DIR)
 def test_integration_commands_run(cli, datafiles):
     project = str(datafiles)
diff --git a/tests/examples/junctions.py b/tests/examples/junctions.py
index b77fcba8a..18bf4da4f 100644
--- a/tests/examples/junctions.py
+++ b/tests/examples/junctions.py
@@ -19,8 +19,6 @@ DATA_DIR = os.path.join(
                     reason='Examples are written for x86-64')
 @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with bubblewrap')
 @pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 @pytest.mark.datafiles(DATA_DIR)
 def test_build(cli, datafiles):
     project = str(datafiles)
@@ -34,8 +32,6 @@ def test_build(cli, datafiles):
                     reason='Examples are written for x86-64')
 @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with bubblewrap')
 @pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 @pytest.mark.datafiles(DATA_DIR)
 def test_shell_call_hello(cli, datafiles):
     project = str(datafiles)
diff --git a/tests/examples/running-commands.py b/tests/examples/running-commands.py
index ce81da0c8..1c419d524 100644
--- a/tests/examples/running-commands.py
+++ b/tests/examples/running-commands.py
@@ -19,8 +19,6 @@ DATA_DIR = os.path.join(
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
 @pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_running_commands_build(cli, datafiles):
     project = str(datafiles)
 
@@ -33,8 +31,6 @@ def test_running_commands_build(cli, datafiles):
                     reason='Examples are written for x86-64')
 @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
 @pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 @pytest.mark.datafiles(DATA_DIR)
 def test_running_commands_run(cli, datafiles):
     project = str(datafiles)
diff --git a/tests/integration/artifact.py b/tests/integration/artifact.py
index 6ad0f8477..4180bf6fd 100644
--- a/tests/integration/artifact.py
+++ b/tests/integration/artifact.py
@@ -49,8 +49,6 @@ DATA_DIR = os.path.join(
 # Dse this really need a sandbox?
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_cache_buildtrees(cli, tmpdir, datafiles):
     project = str(datafiles)
     element_name = 'autotools/amhello.bst'
diff --git a/tests/integration/autotools.py b/tests/integration/autotools.py
index 250ab90d1..c985e8e6c 100644
--- a/tests/integration/autotools.py
+++ b/tests/integration/autotools.py
@@ -22,8 +22,6 @@ DATA_DIR = os.path.join(
 # amhello project for this.
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_autotools_build(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -68,8 +66,6 @@ def test_autotools_confroot_build(cli, datafiles):
 # Test running an executable built with autotools
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_autotools_run(cli, datafiles):
     project = str(datafiles)
     element_name = 'autotools/amhello.bst'
diff --git a/tests/integration/cachedfail.py b/tests/integration/cachedfail.py
index 4a469e21a..e3b5b2796 100644
--- a/tests/integration/cachedfail.py
+++ b/tests/integration/cachedfail.py
@@ -39,8 +39,6 @@ DATA_DIR = os.path.join(
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_build_checkout_cached_fail(cli, datafiles):
     project = str(datafiles)
     element_path = os.path.join(project, 'elements', 'element.bst')
diff --git a/tests/integration/cmake.py b/tests/integration/cmake.py
index 3cb56ab5c..0fe91ed6d 100644
--- a/tests/integration/cmake.py
+++ b/tests/integration/cmake.py
@@ -20,8 +20,6 @@ DATA_DIR = os.path.join(
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_cmake_build(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -55,8 +53,6 @@ def test_cmake_confroot_build(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_cmake_run(cli, datafiles):
     project = str(datafiles)
     element_name = 'cmake/cmakehello.bst'
diff --git a/tests/integration/compose.py b/tests/integration/compose.py
index dc8f4f858..3562ed94b 100644
--- a/tests/integration/compose.py
+++ b/tests/integration/compose.py
@@ -80,8 +80,6 @@ def create_compose_element(name, path, config=None):
                     '/usr/share/doc/amhello/README'])
 ])
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_compose_include(cli, datafiles, include_domains,
                          exclude_domains, expected):
     project = str(datafiles)
diff --git a/tests/integration/make.py b/tests/integration/make.py
index 4678c5319..664e7ca7a 100644
--- a/tests/integration/make.py
+++ b/tests/integration/make.py
@@ -22,7 +22,6 @@ DATA_DIR = os.path.join(
 # makehello project for this.
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 def test_make_build(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -41,7 +40,6 @@ def test_make_build(cli, datafiles):
 # Test running an executable built with make
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 def test_make_run(cli, datafiles):
     project = str(datafiles)
     element_name = 'make/makehello.bst'
diff --git a/tests/integration/manual.py b/tests/integration/manual.py
index bcfcaaf41..2ac7f74d0 100644
--- a/tests/integration/manual.py
+++ b/tests/integration/manual.py
@@ -36,8 +36,6 @@ def create_manual_element(name, path, config, variables, environment):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_manual_element(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -72,8 +70,6 @@ strip
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_manual_element_environment(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -104,8 +100,6 @@ def test_manual_element_environment(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_manual_element_noparallel(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -141,8 +135,6 @@ def test_manual_element_noparallel(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_manual_element_logging(cli, datafiles):
     project = str(datafiles)
     element_path = os.path.join(project, 'elements')
diff --git a/tests/integration/messages.py b/tests/integration/messages.py
index 0313a6347..42725fc5b 100644
--- a/tests/integration/messages.py
+++ b/tests/integration/messages.py
@@ -41,8 +41,6 @@ DATA_DIR = os.path.join(
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_disable_message_lines(cli, datafiles):
     project = str(datafiles)
     element_path = os.path.join(project, 'elements')
diff --git a/tests/integration/pip_element.py b/tests/integration/pip_element.py
index 85a922c00..da0badcb3 100644
--- a/tests/integration/pip_element.py
+++ b/tests/integration/pip_element.py
@@ -25,8 +25,6 @@ DATA_DIR = os.path.join(
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_pip_build(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -64,8 +62,6 @@ def test_pip_build(cli, datafiles):
 # Test running an executable built with pip
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_pip_run(cli, datafiles):
     # Create and build our test element
     test_pip_build(cli, datafiles)
@@ -80,8 +76,6 @@ def test_pip_run(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_pip_element_should_install_pip_deps(cli, datafiles, setup_pypi_repo):
     project = str(datafiles)
     elements_path = os.path.join(project, 'elements')
diff --git a/tests/integration/pip_source.py b/tests/integration/pip_source.py
index c8f997800..c221910a6 100644
--- a/tests/integration/pip_source.py
+++ b/tests/integration/pip_source.py
@@ -140,8 +140,6 @@ def test_pip_source_import_requirements_files(cli, datafiles, setup_pypi_repo):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_pip_source_build(cli, datafiles, setup_pypi_repo):
     project = str(datafiles)
     element_path = os.path.join(project, 'elements')
diff --git a/tests/integration/pullbuildtrees.py b/tests/integration/pullbuildtrees.py
index 437b3e0a5..af9186b1b 100644
--- a/tests/integration/pullbuildtrees.py
+++ b/tests/integration/pullbuildtrees.py
@@ -37,8 +37,6 @@ def default_state(cli, tmpdir, share):
 @pytest.mark.integration
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_pullbuildtrees(cli2, tmpdir, datafiles):
     project = str(datafiles)
     element_name = 'autotools/amhello.bst'
diff --git a/tests/integration/script.py b/tests/integration/script.py
index 04e70af32..dc893f679 100644
--- a/tests/integration/script.py
+++ b/tests/integration/script.py
@@ -40,8 +40,6 @@ def create_script_element(name, path, config=None, variables=None):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_script(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -70,8 +68,6 @@ def test_script(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_script_root(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -160,8 +156,6 @@ def test_script_cwd(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_script_layout(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -181,8 +175,6 @@ def test_script_layout(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_regression_cache_corruption(cli, datafiles):
     project = str(datafiles)
     checkout_original = os.path.join(cli.directory, 'checkout-original')
@@ -213,8 +205,6 @@ def test_regression_cache_corruption(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_regression_tmpdir(cli, datafiles):
     project = str(datafiles)
     element_name = 'script/tmpdir.bst'
@@ -225,8 +215,6 @@ def test_regression_tmpdir(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_regression_cache_corruption_2(cli, datafiles):
     project = str(datafiles)
     checkout_original = os.path.join(cli.directory, 'checkout-original')
diff --git a/tests/integration/shell.py b/tests/integration/shell.py
index 8ea5d5e69..e4c521815 100644
--- a/tests/integration/shell.py
+++ b/tests/integration/shell.py
@@ -57,8 +57,6 @@ def execute_shell(cli, project, command, *, config=None, mount=None, element='ba
 # executable
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_shell(cli, datafiles):
     project = str(datafiles)
 
@@ -70,8 +68,6 @@ def test_shell(cli, datafiles):
 # Test running an executable directly
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_executable(cli, datafiles):
     project = str(datafiles)
 
@@ -84,8 +80,7 @@ def test_executable(cli, datafiles):
 @pytest.mark.parametrize("animal", [("Horse"), ("Pony")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
+# This test seems to fail or pass depending on if this file is run or the hole test suite
 def test_env_assign(cli, datafiles, animal):
     project = str(datafiles)
     expected = animal + '\n'
@@ -106,8 +101,7 @@ def test_env_assign(cli, datafiles, animal):
 @pytest.mark.parametrize("animal", [("Horse"), ("Pony")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
+# This test seems to fail or pass depending on if this file is run or the hole test suite
 def test_env_assign_expand_host_environ(cli, datafiles, animal):
     project = str(datafiles)
     expected = 'The animal is: {}\n'.format(animal)
@@ -131,8 +125,7 @@ def test_env_assign_expand_host_environ(cli, datafiles, animal):
 @pytest.mark.parametrize("animal", [("Horse"), ("Pony")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
+# This test seems to faili or pass depending on if this file is run or the hole test suite
 def test_env_assign_isolated(cli, datafiles, animal):
     project = str(datafiles)
     result = execute_shell(cli, project, ['/bin/sh', '-c', 'echo ${ANIMAL}'], isolate=True, config={
@@ -151,8 +144,6 @@ def test_env_assign_isolated(cli, datafiles, animal):
 # /bin/sh)
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_no_shell(cli, datafiles):
     project = str(datafiles)
     element_path = os.path.join(project, 'elements')
@@ -235,7 +226,6 @@ def test_host_files_expand_environ(cli, datafiles, path):
 @pytest.mark.parametrize("path", [("/etc/pony.conf"), ("/usr/share/pony/pony.txt")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 def test_isolated_no_mount(cli, datafiles, path):
     project = str(datafiles)
     ponyfile = os.path.join(project, 'files', 'shell-mount', 'pony.txt')
@@ -259,7 +249,6 @@ def test_isolated_no_mount(cli, datafiles, path):
 @pytest.mark.parametrize("optional", [("mandatory"), ("optional")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 def test_host_files_missing(cli, datafiles, optional):
     project = str(datafiles)
     ponyfile = os.path.join(project, 'files', 'shell-mount', 'horsy.txt')
@@ -372,8 +361,6 @@ def test_sysroot(cli, tmpdir, datafiles):
 # Test system integration commands can access devices in /dev
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_integration_devices(cli, datafiles):
     project = str(datafiles)
     element_name = 'integration.bst'
@@ -421,8 +408,6 @@ def test_integration_external_workspace(cli, tmpdir_factory, datafiles, build_sh
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_integration_partial_artifact(cli, datafiles, tmpdir, integration_cache):
 
     project = str(datafiles)
diff --git a/tests/integration/shellbuildtrees.py b/tests/integration/shellbuildtrees.py
index 4f4d8b0b5..81a279479 100644
--- a/tests/integration/shellbuildtrees.py
+++ b/tests/integration/shellbuildtrees.py
@@ -24,8 +24,6 @@ DATA_DIR = os.path.join(
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_buildtree_staged(cli_integration, datafiles):
     # We can only test the non interacitve case
     # The non interactive case defaults to not using buildtrees
@@ -44,8 +42,6 @@ def test_buildtree_staged(cli_integration, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_buildtree_staged_forced_true(cli_integration, datafiles):
     # Test that if we ask for a build tree it is there.
     project = str(datafiles)
@@ -63,8 +59,6 @@ def test_buildtree_staged_forced_true(cli_integration, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_buildtree_staged_warn_empty_cached(cli_integration, tmpdir, datafiles):
     # Test that if we stage a cached and empty buildtree, we warn the user.
     project = str(datafiles)
@@ -96,8 +90,6 @@ def test_buildtree_staged_warn_empty_cached(cli_integration, tmpdir, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_buildtree_staged_if_available(cli_integration, datafiles):
     # Test that a build tree can be correctly detected.
     project = str(datafiles)
@@ -115,8 +107,6 @@ def test_buildtree_staged_if_available(cli_integration, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_buildtree_staged_forced_false(cli_integration, datafiles):
     # Test that if we ask not to have a build tree it is not there
     project = str(datafiles)
@@ -135,8 +125,6 @@ def test_buildtree_staged_forced_false(cli_integration, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_buildtree_from_failure(cli_integration, datafiles):
     # Test that we can use a build tree after a failure
     project = str(datafiles)
@@ -179,8 +167,6 @@ def test_buildtree_from_failure_option_never(cli_integration, tmpdir, datafiles)
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_buildtree_from_failure_option_always(cli_integration, tmpdir, datafiles):
 
     project = str(datafiles)
@@ -208,8 +194,6 @@ def test_buildtree_from_failure_option_always(cli_integration, tmpdir, datafiles
 # This is to roughly simulate remote execution
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_buildtree_pulled(cli, tmpdir, datafiles):
     project = str(datafiles)
     element_name = 'build-shell/buildtree.bst'
@@ -243,8 +227,6 @@ def test_buildtree_pulled(cli, tmpdir, datafiles):
 # This test checks for correct behaviour if a buildtree is not present in the local cache.
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_buildtree_options(cli, tmpdir, datafiles):
     project = str(datafiles)
     element_name = 'build-shell/buildtree.bst'
@@ -328,8 +310,6 @@ def test_buildtree_options(cli, tmpdir, datafiles):
 # Tests running pull and pull-buildtree options at the same time.
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_pull_buildtree_pulled(cli, tmpdir, datafiles):
     project = str(datafiles)
     element_name = 'build-shell/buildtree.bst'
diff --git a/tests/integration/sockets.py b/tests/integration/sockets.py
index 246e48595..763238baf 100644
--- a/tests/integration/sockets.py
+++ b/tests/integration/sockets.py
@@ -18,8 +18,6 @@ DATA_DIR = os.path.join(
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_builddir_socket_ignored(cli, datafiles):
     project = str(datafiles)
     element_name = 'sockets/make-builddir-socket.bst'
@@ -30,8 +28,6 @@ def test_builddir_socket_ignored(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_install_root_socket_ignored(cli, datafiles):
     project = str(datafiles)
     element_name = 'sockets/make-install-root-socket.bst'
diff --git a/tests/integration/source-determinism.py b/tests/integration/source-determinism.py
index 0ac954f4f..70c4b79de 100644
--- a/tests/integration/source-determinism.py
+++ b/tests/integration/source-determinism.py
@@ -32,8 +32,6 @@ def create_test_directory(*path, mode=0o644):
 @pytest.mark.integration
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_deterministic_source_local(cli, tmpdir, datafiles):
     """Only user rights should be considered for local source.
     """
diff --git a/tests/integration/stack.py b/tests/integration/stack.py
index c6eaba3c4..9d6b38345 100644
--- a/tests/integration/stack.py
+++ b/tests/integration/stack.py
@@ -19,8 +19,6 @@ DATA_DIR = os.path.join(
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_stack(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
diff --git a/tests/integration/symlinks.py b/tests/integration/symlinks.py
index c62bad586..85bbc53fd 100644
--- a/tests/integration/symlinks.py
+++ b/tests/integration/symlinks.py
@@ -19,8 +19,6 @@ DATA_DIR = os.path.join(
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_absolute_symlinks(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -42,8 +40,6 @@ def test_absolute_symlinks(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_disallow_overlaps_inside_symlink_with_dangling_target(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -59,8 +55,6 @@ def test_disallow_overlaps_inside_symlink_with_dangling_target(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_detect_symlink_overlaps_pointing_outside_sandbox(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -80,8 +74,6 @@ def test_detect_symlink_overlaps_pointing_outside_sandbox(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-# Not stricked xfail as only fails in CI
 def test_symlink_in_sandbox_path(cli, datafiles):
     project = str(datafiles)
     element_name = 'symlinks/link-on-path-use.bst'