Merge branch 'juerg/drop-chroot-sandbox' into 'master'

Drop chroot sandboxing backend

See merge request BuildStream/buildstream!1777

9 files changed, 0 insertions, 407 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d535c184e..002146f9b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -114,25 +114,6 @@ overnight-fedora-30-aarch64:
   only:
   - schedules
 
-tests-unix:
-  # Use fedora here, to a) run a test on fedora and b) ensure that we
-  # can get rid of ostree - this is not possible with debian-8
-  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:31-${DOCKER_IMAGE_VERSION}
-  <<: *tests
-  variables:
-    BST_FORCE_SANDBOX: "chroot"
-
-  script:
-
-    # We remove the Bubblewrap and OSTree packages here so that we catch any
-    # codepaths that try to use them. Removing OSTree causes fuse-libs to
-    # disappear unless we mark it as user-installed.
-    - dnf mark install fuse-libs systemd-udev
-    - dnf erase -y bubblewrap ostree
-
-    # Since the unix platform is required to run as root, no user change required
-    - ${TEST_COMMAND}
-
 tests-buildbox-run:
   image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:31-${DOCKER_IMAGE_VERSION}
   <<: *tests
@@ -472,7 +453,6 @@ coverage:
   - tests-fedora-update-deps
   - tests-remote-execution
   - tests-ubuntu-18.04
-  - tests-unix
   - tests-userchroot
   except:
   - schedules
diff --git a/src/buildstream/_platform/linux.py b/src/buildstream/_platform/linux.py
index 05e55e0f5..c5192c86d 100644
--- a/src/buildstream/_platform/linux.py
+++ b/src/buildstream/_platform/linux.py
@@ -31,7 +31,6 @@ class Linux(Platform):
         sandbox_setups = {
             "bwrap": self._setup_bwrap_sandbox,
             "buildbox-run": self.setup_buildboxrun_sandbox,
-            "chroot": self._setup_chroot_sandbox,
             "dummy": self._setup_dummy_sandbox,
         }
 
@@ -107,23 +106,3 @@ class Linux(Platform):
         self.check_sandbox_config = self._check_sandbox_config_bwrap
         self.create_sandbox = self._create_bwrap_sandbox
         return True
-
-    # Chroot sandbox methods
-    def _check_sandbox_config_chroot(self, config):
-        from ..sandbox._sandboxchroot import SandboxChroot
-
-        return SandboxChroot.check_sandbox_config(self, config)
-
-    @staticmethod
-    def _create_chroot_sandbox(*args, **kwargs):
-        from ..sandbox._sandboxchroot import SandboxChroot
-
-        return SandboxChroot(*args, **kwargs)
-
-    def _setup_chroot_sandbox(self):
-        from ..sandbox._sandboxchroot import SandboxChroot
-
-        self._check_sandbox(SandboxChroot)
-        self.check_sandbox_config = self._check_sandbox_config_chroot
-        self.create_sandbox = Linux._create_chroot_sandbox
-        return True
diff --git a/src/buildstream/sandbox/_sandboxchroot.py b/src/buildstream/sandbox/_sandboxchroot.py
deleted file mode 100644
index 1805131b1..000000000
--- a/src/buildstream/sandbox/_sandboxchroot.py
+++ /dev/null
@@ -1,354 +0,0 @@
-#
-#  Copyright (C) 2017 Codethink Limited
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU Lesser General Public
-#  License as published by the Free Software Foundation; either
-#  version 2 of the License, or (at your option) any later version.
-#
-#  This library is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
-#  Lesser General Public License for more details.
-#
-#  You should have received a copy of the GNU Lesser General Public
-#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
-#
-#  Authors:
-#        Tristan Maat <tristan.maat@codethink.co.uk>
-#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
-
-import os
-import sys
-import stat
-import signal
-import subprocess
-from contextlib import contextmanager, ExitStack
-import psutil
-
-from .._exceptions import SandboxError, PlatformError
-from .. import utils
-from .. import _signals
-from ._mounter import Mounter
-from ._mount import MountMap
-from . import Sandbox, SandboxFlags, SandboxCommandError
-
-
-class SandboxChroot(Sandbox):
-    _FUSE_MOUNT_OPTIONS = {"dev": True}
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-        uid = self._get_config().build_uid
-        gid = self._get_config().build_gid
-        if uid != 0 or gid != 0:
-            raise SandboxError(
-                "Chroot sandboxes cannot specify a non-root uid/gid "
-                "({},{} were supplied via config)".format(uid, gid)
-            )
-
-        self.mount_map = None
-
-    @classmethod
-    def check_available(cls):
-        cls._uid = os.getuid()
-        cls._gid = os.getgid()
-
-        available = cls._uid == 0
-        if not available:
-            cls._dummy_reasons += ["uid is not 0"]
-            raise SandboxError("can not run chroot if uid is not 0")
-
-    @classmethod
-    def check_sandbox_config(cls, local_platform, config):
-        # With the chroot sandbox, the UID/GID in the sandbox
-        # will match the host UID/GID (typically 0/0).
-        if config.build_uid != cls._uid or config.build_gid != cls._gid:
-            return False
-
-        host_os = local_platform.get_host_os()
-        host_arch = local_platform.get_host_arch()
-        # Check host os and architecture match
-        if config.build_os != host_os:
-            raise PlatformError("Configured and host OS don't match.")
-        if config.build_arch != host_arch:
-            raise PlatformError("Configured and host architecture don't match.")
-
-        return True
-
-    def _run(self, command, flags, *, cwd, env):
-
-        if not self._has_command(command[0], env):
-            raise SandboxCommandError(
-                "Staged artifacts do not provide command " "'{}'".format(command[0]), reason="missing-command"
-            )
-
-        stdout, stderr = self._get_output()
-
-        # Create the mount map, this will tell us where
-        # each mount point needs to be mounted from and to
-        self.mount_map = MountMap(self, flags & SandboxFlags.ROOT_READ_ONLY, self._FUSE_MOUNT_OPTIONS)
-
-        # Create a sysroot and run the command inside it
-        with ExitStack() as stack:
-            os.makedirs("/var/run/buildstream", exist_ok=True)
-
-            # FIXME: While we do not currently do anything to prevent
-            # network access, we also don't copy /etc/resolv.conf to
-            # the new rootfs.
-            #
-            # This effectively disables network access, since DNs will
-            # never resolve, so anything a normal process wants to do
-            # will fail. Malicious processes could gain rights to
-            # anything anyway.
-            #
-            # Nonetheless a better solution could perhaps be found.
-
-            rootfs = stack.enter_context(utils._tempdir(dir="/var/run/buildstream"))
-            stack.enter_context(self.create_devices(self._root, flags))
-            stack.enter_context(self.mount_dirs(rootfs, flags, stdout, stderr))
-
-            if flags & SandboxFlags.INTERACTIVE:
-                stdin = sys.stdin
-            else:
-                stdin = stack.enter_context(open(os.devnull, "r"))
-
-            # Ensure the cwd exists
-            if cwd is not None:
-                workdir = os.path.join(rootfs, cwd.lstrip(os.sep))
-                os.makedirs(workdir, exist_ok=True)
-            status = self.chroot(rootfs, command, stdin, stdout, stderr, cwd, env, flags)
-
-        self._vdir._mark_changed()
-        return status
-
-    # chroot()
-    #
-    # A helper function to chroot into the rootfs.
-    #
-    # Args:
-    #    rootfs (str): The path of the sysroot to chroot into
-    #    command (list): The command to execute in the chroot env
-    #    stdin (file): The stdin
-    #    stdout (file): The stdout
-    #    stderr (file): The stderr
-    #    cwd (str): The current working directory
-    #    env (dict): The environment variables to use while executing the command
-    #    flags (:class:`SandboxFlags`): The flags to enable on the sandbox
-    #
-    # Returns:
-    #    (int): The exit code of the executed command
-    #
-    def chroot(self, rootfs, command, stdin, stdout, stderr, cwd, env, flags):
-        def kill_proc():
-            if process:
-                # First attempt to gracefully terminate
-                proc = psutil.Process(process.pid)
-                proc.terminate()
-
-                try:
-                    proc.wait(20)
-                except psutil.TimeoutExpired:
-                    utils._kill_process_tree(process.pid)
-
-        def suspend_proc():
-            group_id = os.getpgid(process.pid)
-            os.killpg(group_id, signal.SIGSTOP)
-
-        def resume_proc():
-            group_id = os.getpgid(process.pid)
-            os.killpg(group_id, signal.SIGCONT)
-
-        try:
-            with _signals.suspendable(suspend_proc, resume_proc), _signals.terminator(kill_proc):
-                process = subprocess.Popen(  # pylint: disable=subprocess-popen-preexec-fn
-                    command,
-                    close_fds=True,
-                    cwd=os.path.join(rootfs, cwd.lstrip(os.sep)),
-                    env=env,
-                    stdin=stdin,
-                    stdout=stdout,
-                    stderr=stderr,
-                    # If you try to put gtk dialogs here Tristan (either)
-                    # will personally scald you
-                    preexec_fn=lambda: (os.chroot(rootfs), os.chdir(cwd)),
-                    start_new_session=flags & SandboxFlags.INTERACTIVE,
-                )
-
-                # Wait for the child process to finish, ensuring that
-                # a SIGINT has exactly the effect the user probably
-                # expects (i.e. let the child process handle it).
-                try:
-                    while True:
-                        try:
-                            _, status = os.waitpid(process.pid, 0)
-                            # If the process exits due to a signal, we
-                            # brutally murder it to avoid zombies
-                            if not os.WIFEXITED(status):
-                                utils._kill_process_tree(process.pid)
-
-                        # Unlike in the bwrap case, here only the main
-                        # process seems to receive the SIGINT. We pass
-                        # on the signal to the child and then continue
-                        # to wait.
-                        except KeyboardInterrupt:
-                            process.send_signal(signal.SIGINT)
-                            continue
-
-                        break
-                # If we can't find the process, it has already died of
-                # its own accord, and therefore we don't need to check
-                # or kill anything.
-                except psutil.NoSuchProcess:
-                    pass
-
-                # Return the exit code - see the documentation for
-                # os.WEXITSTATUS to see why this is required.
-                if os.WIFEXITED(status):
-                    code = os.WEXITSTATUS(status)
-                else:
-                    code = -1
-
-        except subprocess.SubprocessError as e:
-            # Exceptions in preexec_fn are simply reported as
-            # 'Exception occurred in preexec_fn', turn these into
-            # a more readable message.
-            if str(e) == "Exception occurred in preexec_fn.":
-                raise SandboxError(
-                    "Could not chroot into {} or chdir into {}. "
-                    "Ensure you are root and that the relevant directory exists.".format(rootfs, cwd)
-                ) from e
-
-            # Otherwise, raise a more general error
-            raise SandboxError("Could not run command {}: {}".format(command, e)) from e
-
-        return code
-
-    # create_devices()
-    #
-    # Create the nodes in /dev/ usually required for builds (null,
-    # none, etc.)
-    #
-    # Args:
-    #    rootfs (str): The path of the sysroot to prepare
-    #    flags (:class:`.SandboxFlags`): The sandbox flags
-    #
-    @contextmanager
-    def create_devices(self, rootfs, flags):
-
-        devices = []
-        # When we are interactive, we'd rather mount /dev due to the
-        # sheer number of devices
-        if not flags & SandboxFlags.INTERACTIVE:
-
-            for device in Sandbox.DEVICES:
-                location = os.path.join(rootfs, device.lstrip(os.sep))
-                os.makedirs(os.path.dirname(location), exist_ok=True)
-                try:
-                    if os.path.exists(location):
-                        os.remove(location)
-
-                    devices.append(self.mknod(device, location))
-                except OSError as err:
-                    if err.errno == 1:
-                        raise SandboxError(
-                            "Permission denied while creating device node: {}.".format(err)
-                            + "BuildStream reqiures root permissions for these setttings."
-                        )
-
-                    raise
-
-        yield
-
-        for device in devices:
-            os.remove(device)
-
-    # mount_dirs()
-    #
-    # Mount paths required for the command.
-    #
-    # Args:
-    #    rootfs (str): The path of the sysroot to prepare
-    #    flags (:class:`.SandboxFlags`): The sandbox flags
-    #    stdout (file): The stdout
-    #    stderr (file): The stderr
-    #
-    @contextmanager
-    def mount_dirs(self, rootfs, flags, stdout, stderr):
-
-        # FIXME: This should probably keep track of potentially
-        #        already existing files a la _sandboxwrap.py:239
-
-        @contextmanager
-        def mount_point(point, **kwargs):
-            mount_source_overrides = self._get_mount_sources()
-            if point in mount_source_overrides:  # pylint: disable=consider-using-get
-                mount_source = mount_source_overrides[point]
-            else:
-                mount_source = self.mount_map.get_mount_source(point)
-            mount_point = os.path.join(rootfs, point.lstrip(os.sep))
-
-            with Mounter.bind_mount(mount_point, src=mount_source, stdout=stdout, stderr=stderr, **kwargs):
-                yield
-
-        @contextmanager
-        def mount_src(src, **kwargs):
-            mount_point = os.path.join(rootfs, src.lstrip(os.sep))
-            os.makedirs(mount_point, exist_ok=True)
-
-            with Mounter.bind_mount(mount_point, src=src, stdout=stdout, stderr=stderr, **kwargs):
-                yield
-
-        with ExitStack() as stack:
-            stack.enter_context(self.mount_map.mounted(self))
-
-            stack.enter_context(mount_point("/"))
-
-            if flags & SandboxFlags.INTERACTIVE:
-                stack.enter_context(mount_src("/dev"))
-
-            stack.enter_context(mount_src("/tmp"))
-            stack.enter_context(mount_src("/proc"))
-
-            for mark in self._get_marked_directories():
-                stack.enter_context(mount_point(mark["directory"]))
-
-            # Remount root RO if necessary
-            if flags & flags & SandboxFlags.ROOT_READ_ONLY:
-                root_mount = Mounter.mount(rootfs, stdout=stdout, stderr=stderr, remount=True, ro=True, bind=True)
-                # Since the exit stack has already registered a mount
-                # for this path, we do not need to register another
-                # umount call.
-                root_mount.__enter__()
-
-            yield
-
-    # mknod()
-    #
-    # Create a device node equivalent to the given source node
-    #
-    # Args:
-    #    source (str): Path of the device to mimic (e.g. '/dev/null')
-    #    target (str): Location to create the new device in
-    #
-    # Returns:
-    #    target (str): The location of the created node
-    #
-    def mknod(self, source, target):
-        try:
-            dev = os.stat(source)
-            major = os.major(dev.st_rdev)
-            minor = os.minor(dev.st_rdev)
-
-            target_dev = os.makedev(major, minor)
-
-            os.mknod(target, mode=stat.S_IFCHR | dev.st_mode, device=target_dev)
-
-        except PermissionError as e:
-            raise SandboxError("Could not create device {}, ensure that you have root permissions: {}")
-
-        except OSError as e:
-            raise SandboxError("Could not create device {}: {}".format(target, e)) from e
-
-        return target
diff --git a/tests/examples/autotools.py b/tests/examples/autotools.py
index 46b9d117e..c2cd0548b 100644
--- a/tests/examples/autotools.py
+++ b/tests/examples/autotools.py
@@ -16,7 +16,6 @@ DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..",
 # Tests a build of the autotools amhello project on a alpine-linux base runtime
 @pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64")
 @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with sandbox")
-@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This test is not meant to work with chroot sandbox")
 @pytest.mark.datafiles(DATA_DIR)
 def test_autotools_build(cli, datafiles):
     project = str(datafiles)
@@ -47,7 +46,6 @@ def test_autotools_build(cli, datafiles):
 # Test running an executable built with autotools.
 @pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64")
 @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with sandbox")
-@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This test is not meant to work with chroot sandbox")
 @pytest.mark.datafiles(DATA_DIR)
 def test_autotools_run(cli, datafiles):
     project = str(datafiles)
diff --git a/tests/examples/developing.py b/tests/examples/developing.py
index d33a26425..90d33bff1 100644
--- a/tests/examples/developing.py
+++ b/tests/examples/developing.py
@@ -17,7 +17,6 @@ DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..",
 # Test that the project builds successfully
 @pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64")
 @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with SANDBOX")
-@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This is not meant to work with chroot")
 @pytest.mark.datafiles(DATA_DIR)
 def test_autotools_build(cli, datafiles):
     project = str(datafiles)
@@ -36,7 +35,6 @@ def test_autotools_build(cli, datafiles):
 # Test the unmodified hello command works as expected.
 @pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64")
 @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with SANDBOX")
-@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This is not meant to work with chroot")
 @pytest.mark.datafiles(DATA_DIR)
 def test_run_unmodified_hello(cli, datafiles):
     project = str(datafiles)
@@ -69,7 +67,6 @@ def test_open_workspace(cli, tmpdir, datafiles):
 # Test making a change using the workspace
 @pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64")
 @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with SANDBOX")
-@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This is not meant to work with chroot")
 @pytest.mark.datafiles(DATA_DIR)
 def test_make_change_in_workspace(cli, tmpdir, datafiles):
     project = str(datafiles)
diff --git a/tests/examples/integration-commands.py b/tests/examples/integration-commands.py
index 2bdbfb3ec..ad270d077 100644
--- a/tests/examples/integration-commands.py
+++ b/tests/examples/integration-commands.py
@@ -16,7 +16,6 @@ DATA_DIR = os.path.join(
 
 @pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64")
 @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with sandbox")
-@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This test is not meant to work with chroot sandbox")
 @pytest.mark.datafiles(DATA_DIR)
 def test_integration_commands_build(cli, datafiles):
     project = str(datafiles)
@@ -28,7 +27,6 @@ def test_integration_commands_build(cli, datafiles):
 # Test running the executable
 @pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64")
 @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with sandbox")
-@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This test is not meant to work with chroot sandbox")
 @pytest.mark.datafiles(DATA_DIR)
 def test_integration_commands_run(cli, datafiles):
     project = str(datafiles)
diff --git a/tests/examples/junctions.py b/tests/examples/junctions.py
index 28b7a6108..1bfc9cdaa 100644
--- a/tests/examples/junctions.py
+++ b/tests/examples/junctions.py
@@ -15,7 +15,6 @@ DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..",
 # Test that the project builds successfully
 @pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64")
 @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with bubblewrap")
-@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This test is not meant to work with chroot sandbox")
 @pytest.mark.datafiles(DATA_DIR)
 def test_build(cli, datafiles):
     project = str(datafiles)
@@ -27,7 +26,6 @@ def test_build(cli, datafiles):
 # Test the callHello script works as expected.
 @pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64")
 @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with bubblewrap")
-@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This test is not meant to work with chroot sandbox")
 @pytest.mark.datafiles(DATA_DIR)
 def test_shell_call_hello(cli, datafiles):
     project = str(datafiles)
diff --git a/tests/examples/running-commands.py b/tests/examples/running-commands.py
index bf8a9c002..9f60b7237 100644
--- a/tests/examples/running-commands.py
+++ b/tests/examples/running-commands.py
@@ -15,7 +15,6 @@ DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..",
 @pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64")
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with sandbox")
-@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This test is not meant to work with chroot sandbox")
 def test_running_commands_build(cli, datafiles):
     project = str(datafiles)
 
@@ -26,7 +25,6 @@ def test_running_commands_build(cli, datafiles):
 # Test running the executable
 @pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64")
 @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with sandbox")
-@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This test is not meant to work with chroot sandbox")
 @pytest.mark.datafiles(DATA_DIR)
 def test_running_commands_run(cli, datafiles):
     project = str(datafiles)
diff --git a/tests/integration/sandbox.py b/tests/integration/sandbox.py
index ec41f1d92..da0073377 100644
--- a/tests/integration/sandbox.py
+++ b/tests/integration/sandbox.py
@@ -32,7 +32,6 @@ DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
 
 
 @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-@pytest.mark.xfail(HAVE_SANDBOX == "chroot", reason="The chroot sandbox doesn't support shm")
 @pytest.mark.datafiles(DATA_DIR)
 def test_sandbox_shm(cli, datafiles):
     project = str(datafiles)