Merge branch 'juerg/buildbox-run-bubblewrap' into 'master'

Add CI job to test buildbox-run-bubblewrap and buildbox-fuse

Closes #1226

See merge request BuildStream/buildstream!1752

17 files changed, 9 insertions, 330 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8cf593f0a..556be266a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -24,7 +24,7 @@ variables:
   # Our own variables
   # Version of the docker images we should use for all the images.
   # This is taken from buildstream/buildstream-docker-images
-  DOCKER_IMAGE_VERSION: master-100413107
+  DOCKER_IMAGE_VERSION: master-101787517
   PYTEST_ADDOPTS: "--color=yes"
   INTEGRATION_CACHE: "${CI_PROJECT_DIR}/cache/integration-cache"
   PYTEST_ARGS: "--color=yes --integration -n 2"
@@ -118,34 +118,11 @@ tests-unix:
     # Since the unix platform is required to run as root, no user change required
     - ${TEST_COMMAND}
 
-tests-buildbox:
+tests-buildbox-run:
   image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:31-${DOCKER_IMAGE_VERSION}
   <<: *tests
   variables:
-    BST_FORCE_SANDBOX: "buildbox"
-
-  script:
-
-    - dnf install -y fuse3
-
-    # Before buildbox is a first class citizen we need a good install story for users and this test
-    # should mirror that story, for now we build in the test as it is quick and easy.
-
-    # Build and install buildbox
-    - dnf install -y fuse3-devel glibc-static grpc-plugins grpc-devel protobuf-devel cmake gcc gcc-c++ libuuid-devel 
-    - git clone https://gitlab.com/BuildGrid/buildbox/buildbox-fuse.git
-    - cd buildbox-fuse
-    # Pin a specific commit so that any changes to buildbox do not result in unexpected/unannounced buildstream failures
-    - git checkout cdd2b00842c39a8f7162c2ae55bf2cefb925e339
-    - cmake -B build
-    - cmake --build build
-    - cmake --build build --target install
-    - cd ..
-
-    - useradd -Um buildstream
-    - chown -R buildstream:buildstream .
-
-    - su buildstream -c "${TEST_COMMAND}"
+    BST_FORCE_SANDBOX: "buildbox-run"
 
 tests-fedora-missing-deps:
   # Ensure that tests behave nicely while missing bwrap and ostree
diff --git a/src/buildstream/_platform/linux.py b/src/buildstream/_platform/linux.py
index a6215c90f..05e55e0f5 100644
--- a/src/buildstream/_platform/linux.py
+++ b/src/buildstream/_platform/linux.py
@@ -24,14 +24,12 @@ from .. import utils
 from ..sandbox import SandboxDummy
 
 from .platform import Platform
-from .._exceptions import PlatformError
 
 
 class Linux(Platform):
     def _setup_sandbox(self, force_sandbox):
         sandbox_setups = {
             "bwrap": self._setup_bwrap_sandbox,
-            "buildbox": self._setup_buildbox_sandbox,
             "buildbox-run": self.setup_buildboxrun_sandbox,
             "chroot": self._setup_chroot_sandbox,
             "dummy": self._setup_dummy_sandbox,
@@ -129,28 +127,3 @@ class Linux(Platform):
         self.check_sandbox_config = self._check_sandbox_config_chroot
         self.create_sandbox = Linux._create_chroot_sandbox
         return True
-
-    # Buildbox sandbox methods
-    def _check_sandbox_config_buildbox(self, config):
-        from ..sandbox._sandboxbuildbox import SandboxBuildBox
-
-        return SandboxBuildBox.check_sandbox_config(self, config)
-
-    @staticmethod
-    def _create_buildbox_sandbox(*args, **kwargs):
-        from ..sandbox._sandboxbuildbox import SandboxBuildBox
-
-        if kwargs.get("allow_real_directory"):
-            raise PlatformError(
-                "The BuildBox Sandbox does not support real directories.",
-                reason="You are using BuildBox sandbox because BST_FORCE_SANBOX=buildbox",
-            )
-        return SandboxBuildBox(*args, **kwargs)
-
-    def _setup_buildbox_sandbox(self):
-        from ..sandbox._sandboxbuildbox import SandboxBuildBox
-
-        self._check_sandbox(SandboxBuildBox)
-        self.check_sandbox_config = self._check_sandbox_config_buildbox
-        self.create_sandbox = self._create_buildbox_sandbox
-        return True
diff --git a/src/buildstream/sandbox/_sandboxbuildbox.py b/src/buildstream/sandbox/_sandboxbuildbox.py
deleted file mode 100644
index c34d95223..000000000
--- a/src/buildstream/sandbox/_sandboxbuildbox.py
+++ /dev/null
@@ -1,248 +0,0 @@
-#
-#  Copyright (C) 2018 Bloomberg LP
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU Lesser General Public
-#  License as published by the Free Software Foundation; either
-#  version 2 of the License, or (at your option) any later version.
-#
-#  This library is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
-#  Lesser General Public License for more details.
-#
-#  You should have received a copy of the GNU Lesser General Public
-#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
-
-import os
-import sys
-import signal
-import subprocess
-from contextlib import ExitStack
-
-import psutil
-
-from .. import utils, _signals, ProgramNotFoundError
-from . import Sandbox, SandboxFlags, SandboxCommandError
-from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
-from ..storage._casbaseddirectory import CasBasedDirectory
-from .._exceptions import SandboxError
-
-
-# SandboxBuidBox()
-#
-# BuildBox-based sandbox implementation.
-#
-class SandboxBuildBox(Sandbox):
-    def __init__(self, context, project, directory, **kwargs):
-        if kwargs.get("allow_real_directory"):
-            raise SandboxError("BuildBox does not support real directories")
-
-        kwargs["allow_real_directory"] = False
-        super().__init__(context, project, directory, **kwargs)
-
-    @classmethod
-    def check_available(cls):
-        try:
-            utils.get_host_tool("buildbox")
-        except utils.ProgramNotFoundError as Error:
-            cls._dummy_reasons += ["buildbox not found"]
-            raise SandboxError(" and ".join(cls._dummy_reasons), reason="unavailable-local-sandbox") from Error
-
-    @classmethod
-    def check_sandbox_config(cls, platform, config):
-        # Report error for elements requiring non-0 UID/GID
-        # TODO
-        if config.build_uid != 0 or config.build_gid != 0:
-            return False
-
-        # Check host os and architecture match
-        if config.build_os != platform.get_host_os():
-            raise SandboxError("Configured and host OS don't match.")
-        if config.build_arch != platform.get_host_arch():
-            raise SandboxError("Configured and host architecture don't match.")
-
-        return True
-
-    def _run(self, command, flags, *, cwd, env):
-        stdout, stderr = self._get_output()
-
-        root_directory = self.get_virtual_directory()
-        scratch_directory = self._get_scratch_directory()
-
-        if not self._has_command(command[0], env):
-            raise SandboxCommandError(
-                "Staged artifacts do not provide command " "'{}'".format(command[0]), reason="missing-command"
-            )
-
-        # Grab the full path of the buildbox binary
-        try:
-            buildbox_command = [utils.get_host_tool("buildbox")]
-        except ProgramNotFoundError as Err:
-            raise SandboxError(
-                ("BuildBox not on path, you are using the BuildBox sandbox because " "BST_FORCE_SANDBOX=buildbox")
-            ) from Err
-
-        for mark in self._get_marked_directories():
-            path = mark["directory"]
-            assert path.startswith("/") and len(path) > 1
-            root_directory.descend(*path[1:].split(os.path.sep), create=True)
-
-        digest = root_directory._get_digest()
-        with open(os.path.join(scratch_directory, "in"), "wb") as input_digest_file:
-            input_digest_file.write(digest.SerializeToString())
-
-        buildbox_command += ["--local=" + root_directory.cas_cache.casdir]
-        buildbox_command += ["--input-digest=in"]
-        buildbox_command += ["--output-digest=out"]
-
-        common_details = (
-            "BuildBox is a experimental sandbox and does not support the requested feature.\n"
-            "You are using this feature because BST_FORCE_SANDBOX=buildbox."
-        )
-
-        if not flags & SandboxFlags.NETWORK_ENABLED:
-            # TODO
-            self._issue_warning("BuildBox sandbox does not have Networking yet", detail=common_details)
-
-        if cwd is not None:
-            buildbox_command += ["--chdir=" + cwd]
-
-        # In interactive mode, we want a complete devpts inside
-        # the container, so there is a /dev/console and such. In
-        # the regular non-interactive sandbox, we want to hand pick
-        # a minimal set of devices to expose to the sandbox.
-        #
-        if flags & SandboxFlags.INTERACTIVE:
-            # TODO
-            self._issue_warning(
-                "BuildBox sandbox does not fully support BuildStream shells yet", detail=common_details
-            )
-
-        if flags & SandboxFlags.ROOT_READ_ONLY:
-            # TODO
-            self._issue_warning(
-                "BuildBox sandbox does not fully support BuildStream `Read only Root`", detail=common_details
-            )
-
-        # Set UID and GID
-        if not flags & SandboxFlags.INHERIT_UID:
-            # TODO
-            self._issue_warning(
-                "BuildBox sandbox does not fully support BuildStream Inherit UID", detail=common_details
-            )
-
-        os.makedirs(os.path.join(scratch_directory, "mnt"), exist_ok=True)
-        buildbox_command += ["mnt"]
-
-        # Add the command
-        buildbox_command += command
-
-        # Use the MountMap context manager to ensure that any redirected
-        # mounts through fuse layers are in context and ready for buildbox
-        # to mount them from.
-        #
-        with ExitStack() as stack:
-            # Ensure the cwd exists
-            if cwd is not None and len(cwd) > 1:
-                assert cwd.startswith("/")
-                root_directory.descend(*cwd[1:].split(os.path.sep), create=True)
-
-            # If we're interactive, we want to inherit our stdin,
-            # otherwise redirect to /dev/null, ensuring process
-            # disconnected from terminal.
-            if flags & SandboxFlags.INTERACTIVE:
-                stdin = sys.stdin
-            else:
-                stdin = stack.enter_context(open(os.devnull, "r"))
-
-            # Run buildbox !
-            exit_code = self.run_buildbox(
-                buildbox_command,
-                stdin,
-                stdout,
-                stderr,
-                env,
-                interactive=(flags & SandboxFlags.INTERACTIVE),
-                cwd=scratch_directory,
-            )
-
-            if exit_code == 0:
-                with open(os.path.join(scratch_directory, "out"), "rb") as output_digest_file:
-                    output_digest = remote_execution_pb2.Digest()
-                    output_digest.ParseFromString(output_digest_file.read())
-                    self._vdir = CasBasedDirectory(root_directory.cas_cache, digest=output_digest)
-
-        return exit_code
-
-    def run_buildbox(self, argv, stdin, stdout, stderr, env, *, interactive, cwd):
-        def kill_proc():
-            if process:
-                # First attempt to gracefully terminate
-                proc = psutil.Process(process.pid)
-                proc.terminate()
-
-                try:
-                    proc.wait(20)
-                except psutil.TimeoutExpired:
-                    utils._kill_process_tree(process.pid)
-
-        def suspend_proc():
-            group_id = os.getpgid(process.pid)
-            os.killpg(group_id, signal.SIGSTOP)
-
-        def resume_proc():
-            group_id = os.getpgid(process.pid)
-            os.killpg(group_id, signal.SIGCONT)
-
-        with _signals.suspendable(suspend_proc, resume_proc), _signals.terminator(kill_proc):
-            process = subprocess.Popen(
-                argv,
-                close_fds=True,
-                env=env,
-                stdin=stdin,
-                stdout=stdout,
-                stderr=stderr,
-                cwd=cwd,
-                start_new_session=interactive,
-            )
-
-            # Wait for the child process to finish, ensuring that
-            # a SIGINT has exactly the effect the user probably
-            # expects (i.e. let the child process handle it).
-            try:
-                while True:
-                    try:
-                        _, status = os.waitpid(process.pid, 0)
-                        # If the process exits due to a signal, we
-                        # brutally murder it to avoid zombies
-                        if not os.WIFEXITED(status):
-                            utils._kill_process_tree(process.pid)
-
-                    # Unlike in the bwrap case, here only the main
-                    # process seems to receive the SIGINT. We pass
-                    # on the signal to the child and then continue
-                    # to wait.
-                    except KeyboardInterrupt:
-                        process.send_signal(signal.SIGINT)
-                        continue
-
-                    break
-            # If we can't find the process, it has already died of
-            # its own accord, and therefore we don't need to check
-            # or kill anything.
-            except psutil.NoSuchProcess:
-                pass
-
-            # Return the exit code - see the documentation for
-            # os.WEXITSTATUS to see why this is required.
-            if os.WIFEXITED(status):
-                exit_code = os.WEXITSTATUS(status)
-            else:
-                exit_code = -1
-
-        return exit_code
-
-    def _use_cas_based_directory(self):
-        # Always use CasBasedDirectory for BuildBox
-        return True
diff --git a/src/buildstream/testing/_sourcetests/source_determinism.py b/src/buildstream/testing/_sourcetests/source_determinism.py
index f84d18ba2..d51d0e520 100644
--- a/src/buildstream/testing/_sourcetests/source_determinism.py
+++ b/src/buildstream/testing/_sourcetests/source_determinism.py
@@ -50,7 +50,6 @@ def create_test_directory(*path, mode=0o644):
 @pytest.mark.integration
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-@pytest.mark.skipif(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox, Must Fix")
 def test_deterministic_source_umask(cli, tmpdir, datafiles, kind):
     project = str(datafiles)
     element_name = "list.bst"
diff --git a/src/buildstream/testing/_utils/site.py b/src/buildstream/testing/_utils/site.py
index 5507df0f7..f3b45dbf6 100644
--- a/src/buildstream/testing/_utils/site.py
+++ b/src/buildstream/testing/_utils/site.py
@@ -70,12 +70,6 @@ casd_path = utils.get_host_tool("buildbox-casd")
 CASD_SEPARATE_USER = bool(os.stat(casd_path).st_mode & stat.S_ISUID)
 del casd_path
 
-try:
-    utils.get_host_tool("buildbox")
-    HAVE_BUILDBOX = True
-except ProgramNotFoundError:
-    HAVE_BUILDBOX = False
-
 IS_LINUX = os.getenv("BST_FORCE_BACKEND", sys.platform).startswith("linux")
 IS_WSL = IS_LINUX and "Microsoft" in platform.uname().release
 IS_WINDOWS = os.name == "nt"
diff --git a/tests/elements/filter.py b/tests/elements/filter.py
index 0befa46f8..3b38094dc 100644
--- a/tests/elements/filter.py
+++ b/tests/elements/filter.py
@@ -8,7 +8,6 @@ import pytest
 
 from buildstream.testing import create_repo
 from buildstream.testing import cli  # pylint: disable=unused-import
-from buildstream.testing._utils.site import HAVE_SANDBOX
 from buildstream._exceptions import ErrorDomain
 from buildstream import _yaml
 
@@ -28,7 +27,6 @@ def test_filter_include(datafiles, cli, tmpdir):
     assert not os.path.exists(os.path.join(checkout, "bar"))
 
 
-@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox")
 @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
 def test_filter_include_dynamic(datafiles, cli, tmpdir):
     project = str(datafiles)
diff --git a/tests/examples/autotools.py b/tests/examples/autotools.py
index d4c12ecac..46b9d117e 100644
--- a/tests/examples/autotools.py
+++ b/tests/examples/autotools.py
@@ -47,7 +47,6 @@ def test_autotools_build(cli, datafiles):
 # Test running an executable built with autotools.
 @pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64")
 @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with sandbox")
-@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox")
 @pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This test is not meant to work with chroot sandbox")
 @pytest.mark.datafiles(DATA_DIR)
 def test_autotools_run(cli, datafiles):
diff --git a/tests/frontend/configurable_warnings.py b/tests/frontend/configurable_warnings.py
index 472df3eea..488dab233 100644
--- a/tests/frontend/configurable_warnings.py
+++ b/tests/frontend/configurable_warnings.py
@@ -9,7 +9,6 @@ from buildstream.plugin import CoreWarnings
 from buildstream._exceptions import ErrorDomain
 from buildstream import _yaml
 from buildstream.testing.runcli import cli  # pylint: disable=unused-import
-from buildstream.testing._utils.site import HAVE_SANDBOX
 
 TOP_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "configuredwarning")
 
@@ -50,8 +49,6 @@ def build_project(datafiles, fatal_warnings):
     ],
 )
 def test_fatal_warnings(cli, datafiles, element_name, fatal_warnings, expect_fatal, error_domain):
-    if HAVE_SANDBOX == "buildbox" and error_domain != ErrorDomain.STREAM:
-        pytest.xfail()
     project_path = build_project(datafiles, fatal_warnings)
 
     result = cli.run(project=project_path, args=["build", element_name])
diff --git a/tests/integration/autotools.py b/tests/integration/autotools.py
index 7a3b946b2..86a06aa4f 100644
--- a/tests/integration/autotools.py
+++ b/tests/integration/autotools.py
@@ -49,7 +49,6 @@ def test_autotools_build(cli, datafiles):
 # amhello project for this.
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox")
 def test_autotools_confroot_build(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, "checkout")
diff --git a/tests/integration/cmake.py b/tests/integration/cmake.py
index f5d2f6cc9..db0f07531 100644
--- a/tests/integration/cmake.py
+++ b/tests/integration/cmake.py
@@ -33,7 +33,6 @@ def test_cmake_build(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox")
 def test_cmake_confroot_build(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, "checkout")
diff --git a/tests/integration/compose.py b/tests/integration/compose.py
index 5fb79d051..2f38aa66c 100644
--- a/tests/integration/compose.py
+++ b/tests/integration/compose.py
@@ -133,7 +133,6 @@ def test_compose_include(cli, datafiles, include_domains, exclude_domains, expec
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox")
 def test_compose_run_integration(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, "checkout")
diff --git a/tests/integration/interactive_build.py b/tests/integration/interactive_build.py
index 84db69ad9..9c832814d 100644
--- a/tests/integration/interactive_build.py
+++ b/tests/integration/interactive_build.py
@@ -53,7 +53,7 @@ def test_failed_build_quit(element_name, build_session, choice):
 
 
 @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-@pytest.mark.xfail(HAVE_SANDBOX in ["buildbox", "buildbox-run"], reason="Not working with BuildBox")
+@pytest.mark.xfail(HAVE_SANDBOX == "buildbox-run", reason="Not working with BuildBox")
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.parametrize("element_name", ["interactive/failed-build.bst"])
 def test_failed_build_log(element_name, build_session):
@@ -93,7 +93,7 @@ def test_failed_build_shell(element_name, build_session):
     # Quit the shell
     build_session.sendline("exit")
     # Quit the session
-    build_session.expect_exact("Choice: [continue]:")
+    build_session.expect_exact("Choice: [continue]:", timeout=PEXPECT_TIMEOUT_LONG)
     build_session.sendline("quit")
 
     build_session.expect_exact(pexpect.EOF)
diff --git a/tests/integration/sandbox.py b/tests/integration/sandbox.py
index d772f5437..ec41f1d92 100644
--- a/tests/integration/sandbox.py
+++ b/tests/integration/sandbox.py
@@ -32,7 +32,6 @@ DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
 
 
 @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="The buildbox sandbox doesn't support shm")
 @pytest.mark.xfail(HAVE_SANDBOX == "chroot", reason="The chroot sandbox doesn't support shm")
 @pytest.mark.datafiles(DATA_DIR)
 def test_sandbox_shm(cli, datafiles):
diff --git a/tests/integration/script.py b/tests/integration/script.py
index b5e21452c..67dd310bd 100644
--- a/tests/integration/script.py
+++ b/tests/integration/script.py
@@ -91,7 +91,6 @@ def test_script_root(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox")
 @pytest.mark.xfail(
     HAVE_SANDBOX == "buildbox-run", reason="Read-only root directory not supported by buildbox-run",
 )
@@ -121,7 +120,6 @@ def test_script_no_root(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox")
 def test_script_cwd(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, "checkout")
diff --git a/tests/integration/shell.py b/tests/integration/shell.py
index 394edeb1d..cea608016 100644
--- a/tests/integration/shell.py
+++ b/tests/integration/shell.py
@@ -158,7 +158,7 @@ def test_no_shell(cli, datafiles):
 @pytest.mark.parametrize("path", [("/etc/pony.conf"), ("/usr/share/pony/pony.txt")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-@pytest.mark.xfail(HAVE_SANDBOX in ["buildbox", "buildbox-run"], reason="Not working with BuildBox")
+@pytest.mark.xfail(HAVE_SANDBOX == "buildbox-run", reason="Not working with BuildBox")
 def test_host_files(cli, datafiles, path):
     project = str(datafiles)
     ponyfile = os.path.join(project, "files", "shell-mount", "pony.txt")
@@ -173,7 +173,7 @@ def test_host_files(cli, datafiles, path):
 @pytest.mark.parametrize("path", [("/etc"), ("/usr/share/pony")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-@pytest.mark.xfail(HAVE_SANDBOX in ["buildbox", "buildbox-run"], reason="Not working with BuildBox")
+@pytest.mark.xfail(HAVE_SANDBOX == "buildbox-run", reason="Not working with BuildBox")
 def test_host_files_expand_environ(cli, datafiles, path):
     project = str(datafiles)
     hostpath = os.path.join(project, "files", "shell-mount")
@@ -246,7 +246,7 @@ def test_host_files_missing(cli, datafiles, optional):
 @pytest.mark.parametrize("path", [("/etc/pony.conf"), ("/usr/share/pony/pony.txt")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-@pytest.mark.xfail(HAVE_SANDBOX in ["buildbox", "buildbox-run"], reason="Not working with BuildBox")
+@pytest.mark.xfail(HAVE_SANDBOX == "buildbox-run", reason="Not working with BuildBox")
 def test_cli_mount(cli, datafiles, path):
     project = str(datafiles)
     ponyfile = os.path.join(project, "files", "shell-mount", "pony.txt")
@@ -294,7 +294,7 @@ def test_workspace_visible(cli, datafiles):
 # Test that '--sysroot' works
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-@pytest.mark.xfail(HAVE_SANDBOX in ["buildbox", "buildbox-run"], reason="Not working with BuildBox")
+@pytest.mark.xfail(HAVE_SANDBOX == "buildbox-run", reason="Not working with BuildBox")
 def test_sysroot(cli, tmpdir, datafiles):
     project = str(datafiles)
     base_element = "base/base-alpine.bst"
diff --git a/tests/integration/workspace.py b/tests/integration/workspace.py
index 8b4baddef..372db672a 100644
--- a/tests/integration/workspace.py
+++ b/tests/integration/workspace.py
@@ -62,7 +62,6 @@ def test_workspace_mount_on_read_only_directory(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox")
 @pytest.mark.xfail(reason="Incremental builds are currently incompatible with workspace source plugin.")
 def test_workspace_commanddir(cli, datafiles):
     project = str(datafiles)
@@ -318,7 +317,6 @@ def test_incremental_configure_commands_run_only_once(cli, datafiles):
 #
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox")
 def test_workspace_missing_last_successful(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, "workspace")
diff --git a/tests/sourcecache/push.py b/tests/sourcecache/push.py
index 0b7bb9954..771a94ca1 100644
--- a/tests/sourcecache/push.py
+++ b/tests/sourcecache/push.py
@@ -30,7 +30,6 @@ from buildstream._project import Project
 from buildstream import _yaml
 from buildstream.testing import cli  # pylint: disable=unused-import
 from buildstream.testing import create_repo
-from buildstream.testing._utils.site import HAVE_SANDBOX
 from tests.testutils import create_artifact_share, dummy_context
 
 DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
@@ -225,7 +224,6 @@ def test_push_fail(cli, tmpdir, datafiles):
     assert "Pushed" not in res.stderr
 
 
-@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox")
 @pytest.mark.datafiles(DATA_DIR)
 def test_source_push_build_fail(cli, tmpdir, datafiles):
     project_dir = str(datafiles)