Mark buildbox tests as xfail if they require non implemented features

18 files changed, 68 insertions, 19 deletions

diff --git a/src/buildstream/testing/_sourcetests/source_determinism.py b/src/buildstream/testing/_sourcetests/source_determinism.py
index fc8ad9893..d46d38b33 100644
--- a/src/buildstream/testing/_sourcetests/source_determinism.py
+++ b/src/buildstream/testing/_sourcetests/source_determinism.py
@@ -50,6 +50,7 @@ def create_test_directory(*path, mode=0o644):
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.parametrize("kind", [*ALL_REPO_KINDS])
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox, Must Fix')
 def test_deterministic_source_umask(cli, tmpdir, datafiles, kind):
     project = str(datafiles)
     element_name = 'list.bst'
diff --git a/tests/elements/filter.py b/tests/elements/filter.py
index db20529bc..d8370c6bb 100644
--- a/tests/elements/filter.py
+++ b/tests/elements/filter.py
@@ -8,6 +8,7 @@ import pytest
 
 from buildstream.testing import create_repo
 from buildstream.testing import cli  # pylint: disable=unused-import
+from buildstream.testing._utils.site import HAVE_SANDBOX
 from buildstream._exceptions import ErrorDomain
 from buildstream import _yaml
 
@@ -30,6 +31,7 @@ def test_filter_include(datafiles, cli, tmpdir):
     assert not os.path.exists(os.path.join(checkout, "bar"))
 
 
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
 def test_filter_include_dynamic(datafiles, cli, tmpdir):
     project = str(datafiles)
diff --git a/tests/examples/autotools.py b/tests/examples/autotools.py
index ca311c4bb..ecd140718 100644
--- a/tests/examples/autotools.py
+++ b/tests/examples/autotools.py
@@ -6,7 +6,7 @@ import pytest
 
 from buildstream.testing import cli_integration as cli  # pylint: disable=unused-import
 from buildstream.testing.integration import assert_contains
-from buildstream.testing._utils.site import HAVE_BWRAP, IS_LINUX, MACHINE_ARCH
+from buildstream.testing._utils.site import IS_LINUX, MACHINE_ARCH, HAVE_SANDBOX
 
 pytestmark = pytest.mark.integration
 
@@ -18,7 +18,8 @@ DATA_DIR = os.path.join(
 # Tests a build of the autotools amhello project on a alpine-linux base runtime
 @pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
                     reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_BWRAP, reason='Only available on linux with bubblewrap')
+@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
+@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
 @pytest.mark.datafiles(DATA_DIR)
 def test_autotools_build(cli, datafiles):
     project = str(datafiles)
@@ -41,7 +42,9 @@ def test_autotools_build(cli, datafiles):
 # Test running an executable built with autotools.
 @pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
                     reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_BWRAP, reason='Only available on linux with bubblewrap')
+@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
+@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
 @pytest.mark.datafiles(DATA_DIR)
 def test_autotools_run(cli, datafiles):
     project = str(datafiles)
diff --git a/tests/examples/developing.py b/tests/examples/developing.py
index c2b950c01..2f53c8a33 100644
--- a/tests/examples/developing.py
+++ b/tests/examples/developing.py
@@ -6,7 +6,7 @@ import pytest
 
 from buildstream.testing import cli_integration as cli  # pylint: disable=unused-import
 from buildstream.testing.integration import assert_contains
-from buildstream.testing._utils.site import HAVE_BWRAP, IS_LINUX, MACHINE_ARCH
+from buildstream.testing._utils.site import IS_LINUX, MACHINE_ARCH, HAVE_SANDBOX
 import tests.testutils.patch as patch
 
 pytestmark = pytest.mark.integration
@@ -19,7 +19,8 @@ DATA_DIR = os.path.join(
 # Test that the project builds successfully
 @pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
                     reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_BWRAP, reason='Only available on linux with bubblewrap')
+@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with SANDBOX')
+@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This is not meant to work with chroot')
 @pytest.mark.datafiles(DATA_DIR)
 def test_autotools_build(cli, datafiles):
     project = str(datafiles)
@@ -40,7 +41,8 @@ def test_autotools_build(cli, datafiles):
 # Test the unmodified hello command works as expected.
 @pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
                     reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_BWRAP, reason='Only available on linux with bubblewrap')
+@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with SANDBOX')
+@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This is not meant to work with chroot')
 @pytest.mark.datafiles(DATA_DIR)
 def test_run_unmodified_hello(cli, datafiles):
     project = str(datafiles)
@@ -73,7 +75,9 @@ def test_open_workspace(cli, tmpdir, datafiles):
 # Test making a change using the workspace
 @pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
                     reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_BWRAP, reason='Only available on linux with bubblewrap')
+@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with SANDBOX')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
+@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This is not meant to work with chroot')
 @pytest.mark.datafiles(DATA_DIR)
 def test_make_change_in_workspace(cli, tmpdir, datafiles):
     project = str(datafiles)
diff --git a/tests/examples/integration-commands.py b/tests/examples/integration-commands.py
index ad794351f..1ed888b5d 100644
--- a/tests/examples/integration-commands.py
+++ b/tests/examples/integration-commands.py
@@ -5,7 +5,7 @@ import os
 import pytest
 
 from buildstream.testing import cli_integration as cli  # pylint: disable=unused-import
-from buildstream.testing._utils.site import HAVE_BWRAP, IS_LINUX, MACHINE_ARCH
+from buildstream.testing._utils.site import IS_LINUX, MACHINE_ARCH, HAVE_SANDBOX
 
 
 pytestmark = pytest.mark.integration
@@ -16,7 +16,8 @@ DATA_DIR = os.path.join(
 
 @pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
                     reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_BWRAP, reason='Only available on linux with bubblewrap')
+@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
+@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
 @pytest.mark.datafiles(DATA_DIR)
 def test_integration_commands_build(cli, datafiles):
     project = str(datafiles)
@@ -28,7 +29,8 @@ def test_integration_commands_build(cli, datafiles):
 # Test running the executable
 @pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
                     reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_BWRAP, reason='Only available on linux with bubblewrap')
+@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
+@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
 @pytest.mark.datafiles(DATA_DIR)
 def test_integration_commands_run(cli, datafiles):
     project = str(datafiles)
diff --git a/tests/examples/junctions.py b/tests/examples/junctions.py
index bb88e5068..18bf4da4f 100644
--- a/tests/examples/junctions.py
+++ b/tests/examples/junctions.py
@@ -5,7 +5,7 @@ import os
 import pytest
 
 from buildstream.testing import cli_integration as cli  # pylint: disable=unused-import
-from buildstream.testing._utils.site import HAVE_BWRAP, IS_LINUX, MACHINE_ARCH
+from buildstream.testing._utils.site import IS_LINUX, MACHINE_ARCH, HAVE_SANDBOX
 
 pytestmark = pytest.mark.integration
 
@@ -17,7 +17,8 @@ DATA_DIR = os.path.join(
 # Test that the project builds successfully
 @pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
                     reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_BWRAP, reason='Only available on linux with bubblewrap')
+@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with bubblewrap')
+@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
 @pytest.mark.datafiles(DATA_DIR)
 def test_build(cli, datafiles):
     project = str(datafiles)
@@ -29,7 +30,8 @@ def test_build(cli, datafiles):
 # Test the callHello script works as expected.
 @pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
                     reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_BWRAP, reason='Only available on linux with bubblewrap')
+@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with bubblewrap')
+@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
 @pytest.mark.datafiles(DATA_DIR)
 def test_shell_call_hello(cli, datafiles):
     project = str(datafiles)
diff --git a/tests/examples/running-commands.py b/tests/examples/running-commands.py
index 23b3e6467..1c419d524 100644
--- a/tests/examples/running-commands.py
+++ b/tests/examples/running-commands.py
@@ -5,7 +5,7 @@ import os
 import pytest
 
 from buildstream.testing import cli_integration as cli  # pylint: disable=unused-import
-from buildstream.testing._utils.site import HAVE_BWRAP, IS_LINUX, MACHINE_ARCH
+from buildstream.testing._utils.site import IS_LINUX, MACHINE_ARCH, HAVE_SANDBOX
 
 
 pytestmark = pytest.mark.integration
@@ -16,8 +16,9 @@ DATA_DIR = os.path.join(
 
 @pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
                     reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_BWRAP, reason='Only available on linux with bubblewrap')
 @pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
+@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
 def test_running_commands_build(cli, datafiles):
     project = str(datafiles)
 
@@ -28,7 +29,8 @@ def test_running_commands_build(cli, datafiles):
 # Test running the executable
 @pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
                     reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_BWRAP, reason='Only available on linux with bubblewrap')
+@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
+@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
 @pytest.mark.datafiles(DATA_DIR)
 def test_running_commands_run(cli, datafiles):
     project = str(datafiles)
diff --git a/tests/frontend/configurable_warnings.py b/tests/frontend/configurable_warnings.py
index 7936b2f89..38b7f2912 100644
--- a/tests/frontend/configurable_warnings.py
+++ b/tests/frontend/configurable_warnings.py
@@ -9,6 +9,7 @@ from buildstream.plugin import CoreWarnings
 from buildstream._exceptions import ErrorDomain
 from buildstream import _yaml
 from buildstream.testing.runcli import cli  # pylint: disable=unused-import
+from buildstream.testing._utils.site import HAVE_SANDBOX
 
 TOP_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
@@ -45,6 +46,7 @@ def build_project(datafiles, fatal_warnings):
     return project_path
 
 
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 @pytest.mark.datafiles(TOP_DIR)
 @pytest.mark.parametrize("element_name, fatal_warnings, expect_fatal, error_domain", [
     ("corewarn.bst", [CoreWarnings.OVERLAPS], True, ErrorDomain.STREAM),
diff --git a/tests/integration/artifact.py b/tests/integration/artifact.py
index 56c516e67..4180bf6fd 100644
--- a/tests/integration/artifact.py
+++ b/tests/integration/artifact.py
@@ -45,6 +45,8 @@ DATA_DIR = os.path.join(
 # A test to capture the integration of the cachebuildtrees
 # behaviour, which by default is to include the buildtree
 # content of an element on caching.
+
+# Dse this really need a sandbox?
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_cache_buildtrees(cli, tmpdir, datafiles):
diff --git a/tests/integration/autotools.py b/tests/integration/autotools.py
index c4bf429f5..c985e8e6c 100644
--- a/tests/integration/autotools.py
+++ b/tests/integration/autotools.py
@@ -44,6 +44,7 @@ def test_autotools_build(cli, datafiles):
 # amhello project for this.
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_autotools_confroot_build(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
diff --git a/tests/integration/cachedfail.py b/tests/integration/cachedfail.py
index a7509ab3b..e3b5b2796 100644
--- a/tests/integration/cachedfail.py
+++ b/tests/integration/cachedfail.py
@@ -139,6 +139,7 @@ def test_build_depend_on_cached_fail(cli, datafiles):
 
 
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.parametrize("on_error", ("continue", "quit"))
 def test_push_cached_fail(cli, tmpdir, datafiles, on_error):
diff --git a/tests/integration/cmake.py b/tests/integration/cmake.py
index 84ea96af2..0fe91ed6d 100644
--- a/tests/integration/cmake.py
+++ b/tests/integration/cmake.py
@@ -36,6 +36,7 @@ def test_cmake_build(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_cmake_confroot_build(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
diff --git a/tests/integration/make.py b/tests/integration/make.py
index 664e7ca7a..4678c5319 100644
--- a/tests/integration/make.py
+++ b/tests/integration/make.py
@@ -22,6 +22,7 @@ DATA_DIR = os.path.join(
 # makehello project for this.
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 def test_make_build(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -40,6 +41,7 @@ def test_make_build(cli, datafiles):
 # Test running an executable built with make
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 def test_make_run(cli, datafiles):
     project = str(datafiles)
     element_name = 'make/makehello.bst'
diff --git a/tests/integration/script.py b/tests/integration/script.py
index fc57e8744..dc893f679 100644
--- a/tests/integration/script.py
+++ b/tests/integration/script.py
@@ -100,6 +100,7 @@ def test_script_root(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_script_no_root(cli, datafiles):
     project = str(datafiles)
     element_path = os.path.join(project, 'elements')
@@ -123,6 +124,7 @@ def test_script_no_root(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_script_cwd(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
diff --git a/tests/integration/shell.py b/tests/integration/shell.py
index a1f38d879..b0eca2471 100644
--- a/tests/integration/shell.py
+++ b/tests/integration/shell.py
@@ -80,6 +80,8 @@ def test_executable(cli, datafiles):
 @pytest.mark.parametrize("animal", [("Horse"), ("Pony")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
+# This test seems to fail or pass depending on if this file is run or the hole test suite
 def test_env_assign(cli, datafiles, animal):
     project = str(datafiles)
     expected = animal + '\n'
@@ -100,6 +102,8 @@ def test_env_assign(cli, datafiles, animal):
 @pytest.mark.parametrize("animal", [("Horse"), ("Pony")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
+# This test seems to fail or pass depending on if this file is run or the hole test suite
 def test_env_assign_expand_host_environ(cli, datafiles, animal):
     project = str(datafiles)
     expected = 'The animal is: {}\n'.format(animal)
@@ -123,6 +127,8 @@ def test_env_assign_expand_host_environ(cli, datafiles, animal):
 @pytest.mark.parametrize("animal", [("Horse"), ("Pony")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
+# This test seems to faili or pass depending on if this file is run or the hole test suite
 def test_env_assign_isolated(cli, datafiles, animal):
     project = str(datafiles)
     result = execute_shell(cli, project, ['/bin/sh', '-c', 'echo ${ANIMAL}'], isolate=True, config={
@@ -174,6 +180,7 @@ def test_no_shell(cli, datafiles):
 @pytest.mark.parametrize("path", [("/etc/pony.conf"), ("/usr/share/pony/pony.txt")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_host_files(cli, datafiles, path):
     project = str(datafiles)
     ponyfile = os.path.join(project, 'files', 'shell-mount', 'pony.txt')
@@ -195,6 +202,7 @@ def test_host_files(cli, datafiles, path):
 @pytest.mark.parametrize("path", [("/etc"), ("/usr/share/pony")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_host_files_expand_environ(cli, datafiles, path):
     project = str(datafiles)
     hostpath = os.path.join(project, 'files', 'shell-mount')
@@ -221,6 +229,7 @@ def test_host_files_expand_environ(cli, datafiles, path):
 @pytest.mark.parametrize("path", [("/etc/pony.conf"), ("/usr/share/pony/pony.txt")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 def test_isolated_no_mount(cli, datafiles, path):
     project = str(datafiles)
     ponyfile = os.path.join(project, 'files', 'shell-mount', 'pony.txt')
@@ -244,6 +253,7 @@ def test_isolated_no_mount(cli, datafiles, path):
 @pytest.mark.parametrize("optional", [("mandatory"), ("optional")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 def test_host_files_missing(cli, datafiles, optional):
     project = str(datafiles)
     ponyfile = os.path.join(project, 'files', 'shell-mount', 'horsy.txt')
@@ -277,6 +287,7 @@ def test_host_files_missing(cli, datafiles, optional):
 @pytest.mark.parametrize("path", [("/etc/pony.conf"), ("/usr/share/pony/pony.txt")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_cli_mount(cli, datafiles, path):
     project = str(datafiles)
     ponyfile = os.path.join(project, 'files', 'shell-mount', 'pony.txt')
@@ -289,6 +300,7 @@ def test_cli_mount(cli, datafiles, path):
 # Test that we can see the workspace files in a shell
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_workspace_visible(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -323,6 +335,7 @@ def test_workspace_visible(cli, datafiles):
 # Test that '--sysroot' works
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_sysroot(cli, tmpdir, datafiles):
     project = str(datafiles)
     base_element = "base/base-alpine.bst"
@@ -366,6 +379,7 @@ def test_integration_devices(cli, datafiles):
 @pytest.mark.parametrize("build_shell", [("build"), ("nobuild")])
 @pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"])
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_integration_external_workspace(cli, tmpdir_factory, datafiles, build_shell, guess_element):
     tmpdir = tmpdir_factory.mktemp("")
     project = str(datafiles)
diff --git a/tests/integration/shellbuildtrees.py b/tests/integration/shellbuildtrees.py
index a1eecb1eb..81a279479 100644
--- a/tests/integration/shellbuildtrees.py
+++ b/tests/integration/shellbuildtrees.py
@@ -59,8 +59,8 @@ def test_buildtree_staged_forced_true(cli_integration, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-def test_buildtree_staged_warn_non_cached(cli_integration, tmpdir, datafiles):
-    # Test that if we attempt to stage a buildtree that was never cached, we warn the user.
+def test_buildtree_staged_warn_empty_cached(cli_integration, tmpdir, datafiles):
+    # Test that if we stage a cached and empty buildtree, we warn the user.
     project = str(datafiles)
     element_name = 'build-shell/buildtree.bst'
 
diff --git a/tests/integration/workspace.py b/tests/integration/workspace.py
index 78379912c..045f8c490 100644
--- a/tests/integration/workspace.py
+++ b/tests/integration/workspace.py
@@ -20,6 +20,7 @@ DATA_DIR = os.path.join(
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_workspace_mount(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -36,6 +37,7 @@ def test_workspace_mount(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_workspace_commanddir(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -53,6 +55,7 @@ def test_workspace_commanddir(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_workspace_updated_dependency(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -107,6 +110,7 @@ def test_workspace_updated_dependency(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_workspace_update_dependency_failed(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -182,6 +186,7 @@ def test_workspace_update_dependency_failed(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_updated_dependency_nested(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -235,6 +240,7 @@ def test_updated_dependency_nested(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_incremental_configure_commands_run_only_once(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -287,6 +293,7 @@ def test_incremental_configure_commands_run_only_once(cli, datafiles):
 #
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_workspace_missing_last_successful(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
diff --git a/tests/sourcecache/push.py b/tests/sourcecache/push.py
index b0fae616e..ad9653f9d 100644
--- a/tests/sourcecache/push.py
+++ b/tests/sourcecache/push.py
@@ -28,7 +28,7 @@ from buildstream._project import Project
 from buildstream import _yaml
 from buildstream.testing import cli  # pylint: disable=unused-import
 from buildstream.testing import create_repo
-
+from buildstream.testing._utils.site import HAVE_SANDBOX
 from tests.testutils import create_artifact_share, dummy_context
 
 DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
@@ -182,6 +182,7 @@ def test_push_fail(cli, tmpdir, datafiles):
     assert "Pushed" not in res.stderr
 
 
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 @pytest.mark.datafiles(DATA_DIR)
 def test_source_push_build_fail(cli, tmpdir, datafiles):
     project_dir = str(datafiles)