path: root/src/buildstream/sandbox/_sandboxbwrap.py

#
#  Copyright (C) 2016 Codethink Limited
#  Copyright (C) 2019 Bloomberg Finance LP
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU Lesser General Public
#  License as published by the Free Software Foundation; either
#  version 2 of the License, or (at your option) any later version.
#
#  This library is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
#  Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public
#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
#
#  Authors:
#        Andrew Leeming <andrew.leeming@codethink.co.uk>
#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
#        William Salmon <will.salmon@codethink.co.uk>

import collections
import json
import os
import sys
import time
import errno
import signal
import subprocess
import shutil
from contextlib import ExitStack, suppress
from tempfile import TemporaryFile

import psutil

from .._exceptions import SandboxError
from .. import utils, _signals
from . import Sandbox, SandboxFlags, SandboxCommandError
from .. import _site


# SandboxBwrap()
#
# Default bubblewrap based sandbox implementation.
#
class SandboxBwrap(Sandbox):
    _have_good_bwrap = None

    # Minimal set of devices for the sandbox
    DEVICES = ["/dev/full", "/dev/null", "/dev/urandom", "/dev/random", "/dev/zero"]

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.linux32 = kwargs["linux32"]

    @classmethod
    def check_available(cls):
        cls._have_fuse = os.path.exists("/dev/fuse")
        if not cls._have_fuse:
            cls._dummy_reasons += ["Fuse is unavailable"]

        try:
            utils.get_host_tool("bwrap")
        except utils.ProgramNotFoundError as Error:
            cls._bwrap_exists = False
            cls._have_good_bwrap = False
            cls._die_with_parent_available = False
            cls._json_status_available = False
            cls._dummy_reasons += ["Bubblewrap not found"]
            raise SandboxError(" and ".join(cls._dummy_reasons), reason="unavailable-local-sandbox") from Error

        bwrap_version = _site.get_bwrap_version()

        cls._bwrap_exists = True
        cls._have_good_bwrap = (0, 1, 2) <= bwrap_version
        cls._die_with_parent_available = (0, 1, 8) <= bwrap_version
        cls._json_status_available = (0, 3, 2) <= bwrap_version
        if not cls._have_good_bwrap:
            cls._dummy_reasons += ["Bubblewrap is too old"]
            raise SandboxError(" and ".join(cls._dummy_reasons))

        cls._uid = os.geteuid()
        cls._gid = os.getegid()

        cls.user_ns_available = cls._check_user_ns_available()

    @staticmethod
    def _check_user_ns_available():
        # Here, lets check if bwrap is able to create user namespaces,
        # issue a warning if it's not available, and save the state
        # locally so that we can inform the sandbox to not try it
        # later on.
        bwrap = utils.get_host_tool("bwrap")
        try:
            whoami = utils.get_host_tool("whoami")
            output = subprocess.check_output(
                [bwrap, "--ro-bind", "/", "/", "--unshare-user", "--uid", "0", "--gid", "0", whoami,],
                universal_newlines=True,
            ).strip()
        except subprocess.CalledProcessError:
            output = ""
        except utils.ProgramNotFoundError:
            output = ""

        return output == "root"

    @classmethod
    def check_sandbox_config(cls, local_platform, config):
        if cls.user_ns_available:
            # User namespace support allows arbitrary build UID/GID settings.
            pass
        elif config.build_uid != local_platform._uid or config.build_gid != local_platform._gid:
            # Without user namespace support, the UID/GID in the sandbox
            # will match the host UID/GID.
            return False

        host_os = local_platform.get_host_os()
        host_arch = local_platform.get_host_arch()
        if config.build_os != host_os:
            raise SandboxError("Configured and host OS don't match.")
        if config.build_arch != host_arch and not local_platform.can_crossbuild(config):
            raise SandboxError("Configured architecture and host architecture don't match.")

        return True

    def _run(self, command, flags, *, cwd, env):
        stdout, stderr = self._get_output()

        # Allowable access to underlying storage as we're part of the sandbox
        root_directory = self.get_virtual_directory()._get_underlying_directory()

        if not self._has_command(command[0], env):
            raise SandboxCommandError(
                "Staged artifacts do not provide command " "'{}'".format(command[0]), reason="missing-command",
            )

        # NOTE: MountMap transitively imports `_fuse/fuse.py` which raises an
        # EnvironmentError when fuse is not found. Since this module is
        # expected to be imported even in absence of fuse, MountMap is imported
        # here, and not at the top of the module.
        from ._mount import MountMap

        # Create the mount map, this will tell us where
        # each mount point needs to be mounted from and to
        mount_map = MountMap(self, flags & SandboxFlags.ROOT_READ_ONLY)
        root_mount_source = mount_map.get_mount_source("/")

        # start command with linux32 if needed
        if self.linux32:
            bwrap_command = [utils.get_host_tool("linux32")]
        else:
            bwrap_command = []

        # Grab the full path of the bwrap binary
        bwrap_command += [utils.get_host_tool("bwrap")]

        for k, v in env.items():
            bwrap_command += ["--setenv", k, v]
        for k in os.environ.keys() - env.keys():
            bwrap_command += ["--unsetenv", k]

        # Create a new pid namespace, this also ensures that any subprocesses
        # are cleaned up when the bwrap process exits.
        bwrap_command += ["--unshare-pid"]

        # Ensure subprocesses are cleaned up when the bwrap parent dies.
        if self._die_with_parent_available:
            bwrap_command += ["--die-with-parent"]

        # Add in the root filesystem stuff first.
        #
        # The rootfs is mounted as RW initially so that further mounts can be
        # placed on top. If a RO root is required, after all other mounts are
        # complete, root is remounted as RO
        bwrap_command += ["--bind", root_mount_source, "/"]

        if not flags & SandboxFlags.NETWORK_ENABLED:
            bwrap_command += ["--unshare-net"]
            bwrap_command += ["--unshare-uts", "--hostname", "buildstream"]
            bwrap_command += ["--unshare-ipc"]

        # Give it a proc and tmpfs
        bwrap_command += ["--proc", "/proc", "--tmpfs", "/tmp"]

        # In interactive mode, we want a complete devpts inside
        # the container, so there is a /dev/console and such. In
        # the regular non-interactive sandbox, we want to hand pick
        # a minimal set of devices to expose to the sandbox.
        #
        if flags & SandboxFlags.INTERACTIVE:
            bwrap_command += ["--dev", "/dev"]
        else:
            for device in self.DEVICES:
                bwrap_command += ["--dev-bind", device, device]

        # Add bind mounts to any marked directories
        marked_directories = self._get_marked_directories()
        mount_source_overrides = self._get_mount_sources()
        for mark in marked_directories:
            mount_point = mark["directory"]
            if mount_point in mount_source_overrides:  # pylint: disable=consider-using-get
                mount_source = mount_source_overrides[mount_point]
            else:
                mount_source = mount_map.get_mount_source(mount_point)

            # Use --dev-bind for all mounts, this is simply a bind mount which does
            # not restrictive about devices.
            #
            # While it's important for users to be able to mount devices
            # into the sandbox for `bst shell` testing purposes, it is
            # harmless to do in a build environment where the directories
            # we mount just never contain device files.
            #
            bwrap_command += ["--dev-bind", mount_source, mount_point]

        if flags & SandboxFlags.ROOT_READ_ONLY:
            bwrap_command += ["--remount-ro", "/"]

        if cwd is not None:
            bwrap_command += ["--dir", cwd]
            bwrap_command += ["--chdir", cwd]

        # Set UID and GUI
        if self.user_ns_available:
            bwrap_command += ["--unshare-user"]
            if not flags & SandboxFlags.INHERIT_UID:
                uid = self._get_config().build_uid
                gid = self._get_config().build_gid
                bwrap_command += ["--uid", str(uid), "--gid", str(gid)]

        with ExitStack() as stack:
            pass_fds = ()
            # Improve error reporting with json-status if available
            if self._json_status_available:
                json_status_file = stack.enter_context(TemporaryFile())
                pass_fds = (json_status_file.fileno(),)
                bwrap_command += ["--json-status-fd", str(json_status_file.fileno())]

            # Add the command
            bwrap_command += command

            # bwrap might create some directories while being suid
            # and may give them to root gid, if it does, we'll want
            # to clean them up after, so record what we already had
            # there just in case so that we can safely cleanup the debris.
            #
            existing_basedirs = {
                directory: os.path.exists(os.path.join(root_directory, directory))
                for directory in ["tmp", "dev", "proc"]
            }

            # Use the MountMap context manager to ensure that any redirected
            # mounts through fuse layers are in context and ready for bwrap
            # to mount them from.
            #
            stack.enter_context(mount_map.mounted(self))

            # If we're interactive, we want to inherit our stdin,
            # otherwise redirect to /dev/null, ensuring process
            # disconnected from terminal.
            if flags & SandboxFlags.INTERACTIVE:
                stdin = sys.stdin
            else:
                stdin = stack.enter_context(open(os.devnull, "r"))

            # Run bubblewrap !
            exit_code = self.run_bwrap(
                bwrap_command, stdin, stdout, stderr, (flags & SandboxFlags.INTERACTIVE), pass_fds,
            )

            # Cleanup things which bwrap might have left behind, while
            # everything is still mounted because bwrap can be creating
            # the devices on the fuse mount, so we should remove it there.
            if not flags & SandboxFlags.INTERACTIVE:
                for device in self.DEVICES:
                    device_path = os.path.join(root_mount_source, device.lstrip("/"))

                    # This will remove the device in a loop, allowing some
                    # retries in case the device file leaked by bubblewrap is still busy
                    self.try_remove_device(device_path)

            # Remove /tmp, this is a bwrap owned thing we want to be sure
            # never ends up in an artifact
            for basedir in ["tmp", "dev", "proc"]:

                # Skip removal of directories which already existed before
                # launching bwrap
                if existing_basedirs[basedir]:
                    continue

                base_directory = os.path.join(root_mount_source, basedir)

                if flags & SandboxFlags.INTERACTIVE:
                    # Be more lenient in interactive mode here.
                    #
                    # In interactive mode; it's possible that the project shell
                    # configuration has mounted some things below the base
                    # directories, such as /dev/dri, and in this case it's less
                    # important to consider cleanup, as we wont be collecting
                    # this build result and creating an artifact.
                    #
                    # Note: Ideally; we should instead fix upstream bubblewrap to
                    #       cleanup any debris it creates at startup time, and do
                    #       the same ourselves for any directories we explicitly create.
                    #
                    shutil.rmtree(base_directory, ignore_errors=True)
                else:
                    try:
                        os.rmdir(base_directory)
                    except FileNotFoundError:
                        # ignore this, if bwrap cleaned up properly then it's not a problem.
                        #
                        # If the directory was not empty on the other hand, then this is clearly
                        # a bug, bwrap mounted a tempfs here and when it exits, that better be empty.
                        pass

            if self._json_status_available:
                json_status_file.seek(0, 0)
                child_exit_code = None
                # The JSON status file's output is a JSON object per line
                # with the keys present identifying the type of message.
                # The only message relevant to us now is the exit-code of the subprocess.
                for line in json_status_file:
                    with suppress(json.decoder.JSONDecodeError):
                        o = json.loads(line.decode())
                        if isinstance(o, collections.abc.Mapping) and "exit-code" in o:
                            child_exit_code = o["exit-code"]
                            break
                if child_exit_code is None:
                    raise SandboxError(
                        "`bwrap' terminated during sandbox setup with exitcode {}".format(exit_code),
                        reason="bwrap-sandbox-fail",
                    )
                exit_code = child_exit_code

        self._vdir._mark_changed()
        return exit_code

    def run_bwrap(self, argv, stdin, stdout, stderr, interactive, pass_fds):
        # Wrapper around subprocess.Popen() with common settings.
        #
        # This function blocks until the subprocess has terminated.
        #
        # It then returns a tuple of (exit code, stdout output, stderr output).
        # If stdout was not equal to subprocess.PIPE, stdout will be None. Same for
        # stderr.

        # Fetch the process actually launched inside the bwrap sandbox, or the
        # intermediat control bwrap processes.
        #
        # NOTE:
        #   The main bwrap process itself is setuid root and as such we cannot
        #   send it any signals. Since we launch bwrap with --unshare-pid, it's
        #   direct child is another bwrap process which retains ownership of the
        #   pid namespace. This is the right process to kill when terminating.
        #
        #   The grandchild is the binary which we asked bwrap to launch on our
        #   behalf, whatever this binary is, it is the right process to use
        #   for suspending and resuming. In the case that this is a shell, the
        #   shell will be group leader and all build scripts will stop/resume
        #   with that shell.
        #
        def get_user_proc(bwrap_pid, grand_child=False):
            bwrap_proc = psutil.Process(bwrap_pid)
            bwrap_children = bwrap_proc.children()
            if bwrap_children:
                if grand_child:
                    bwrap_grand_children = bwrap_children[0].children()
                    if bwrap_grand_children:
                        return bwrap_grand_children[0]
                else:
                    return bwrap_children[0]
            return None

        def terminate_bwrap():
            if process:
                user_proc = get_user_proc(process.pid)
                if user_proc:
                    user_proc.kill()

        def suspend_bwrap():
            if process:
                user_proc = get_user_proc(process.pid, grand_child=True)
                if user_proc:
                    group_id = os.getpgid(user_proc.pid)
                    os.killpg(group_id, signal.SIGSTOP)

        def resume_bwrap():
            if process:
                user_proc = get_user_proc(process.pid, grand_child=True)
                if user_proc:
                    group_id = os.getpgid(user_proc.pid)
                    os.killpg(group_id, signal.SIGCONT)

        with ExitStack() as stack:

            # We want to launch bwrap in a new session in non-interactive
            # mode so that we handle the SIGTERM and SIGTSTP signals separately
            # from the nested bwrap process, but in interactive mode this
            # causes launched shells to lack job control (we dont really
            # know why that is).
            #
            if interactive:
                new_session = False
            else:
                new_session = True
                stack.enter_context(_signals.suspendable(suspend_bwrap, resume_bwrap))
                stack.enter_context(_signals.terminator(terminate_bwrap))

            process = subprocess.Popen(
                argv,
                # The default is to share file descriptors from the parent process
                # to the subprocess, which is rarely good for sandboxing.
                close_fds=True,
                pass_fds=pass_fds,
                stdin=stdin,
                stdout=stdout,
                stderr=stderr,
                start_new_session=new_session,
            )

            # Wait for the child process to finish, ensuring that
            # a SIGINT has exactly the effect the user probably
            # expects (i.e. let the child process handle it).
            try:
                while True:
                    try:
                        _, status = os.waitpid(process.pid, 0)
                        # If the process exits due to a signal, we
                        # brutally murder it to avoid zombies
                        if not os.WIFEXITED(status):
                            user_proc = get_user_proc(process.pid)
                            if user_proc:
                                utils._kill_process_tree(user_proc.pid)

                    # If we receive a KeyboardInterrupt we continue
                    # waiting for the process since we are in the same
                    # process group and it should also have received
                    # the SIGINT.
                    except KeyboardInterrupt:
                        continue

                    break
            # If we can't find the process, it has already died of its
            # own accord, and therefore we don't need to check or kill
            # anything.
            except psutil.NoSuchProcess:
                pass

            # Return the exit code - see the documentation for
            # os.WEXITSTATUS to see why this is required.
            if os.WIFEXITED(status):
                exit_code = os.WEXITSTATUS(status)
            else:
                exit_code = -1

            if interactive and stdin.isatty():
                # Make this process the foreground process again, otherwise the
                # next read() on stdin will trigger SIGTTIN and stop the process.
                # This is required because the sandboxed process does not have
                # permission to do this on its own (running in separate PID namespace).
                #
                # tcsetpgrp() will trigger SIGTTOU when called from a background
                # process, so ignore it temporarily.
                handler = signal.signal(signal.SIGTTOU, signal.SIG_IGN)
                os.tcsetpgrp(0, os.getpid())
                signal.signal(signal.SIGTTOU, handler)

        return exit_code

    def try_remove_device(self, device_path):

        # Put some upper limit on the tries here
        max_tries = 1000
        tries = 0

        while True:
            try:
                os.unlink(device_path)
            except OSError as e:
                if e.errno == errno.EBUSY:
                    # This happens on some machines, seems there is a race sometimes
                    # after bubblewrap returns and the device files it bind-mounted did
                    # not finish unmounting.
                    #
                    if tries < max_tries:
                        tries += 1
                        time.sleep(1 / 100)
                        continue
                    else:
                        # We've reached the upper limit of tries, bail out now
                        # because something must have went wrong
                        #
                        raise
                elif e.errno == errno.ENOENT:
                    # Bubblewrap cleaned it up for us, no problem if we cant remove it
                    break
                else:
                    # Something unexpected, reraise this error
                    raise
            else:
                # Successfully removed the symlink
                break