diff options
| -rw-r--r-- | buildstream/_sandboxchroot.py | 450 | ||||
| -rw-r--r-- | buildstream/sandbox/__init__.py | 23 | ||||
| -rw-r--r-- | buildstream/sandbox/_mount.py | 125 | ||||
| -rw-r--r-- | buildstream/sandbox/_sandboxbwrap.py (renamed from buildstream/_sandboxbwrap.py) | 123 | ||||
| -rw-r--r-- | buildstream/sandbox/_sandboxchroot.py | 263 | ||||
| -rw-r--r-- | buildstream/sandbox/sandbox.py (renamed from buildstream/sandbox.py) | 119 |
6 files changed, 535 insertions, 568 deletions
diff --git a/buildstream/_sandboxchroot.py b/buildstream/_sandboxchroot.py deleted file mode 100644 index 03bdecff4..000000000 --- a/buildstream/_sandboxchroot.py +++ /dev/null @@ -1,450 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2017 Codethink Limited -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see <http://www.gnu.org/licenses/>. -# -# Authors: -# Tristan Maat <tristan.maat@codethink.co.uk> -# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> - -import os -import sys -import stat -import shutil -import subprocess -from contextlib import contextmanager, ExitStack - -from . import utils -from . import ElementError -from . import Sandbox, SandboxFlags - - -# A class to wrap the `mount` and `umount` commands -class Mount(): - def __init__(self, platform): - self.platform = platform - - def _mount(self, dest, src=None, mount_type=None, - stdout=sys.stdout, stderr=sys.stderr, options=None, - flags=None): - - argv = [utils.get_host_tool('mount')] - if mount_type: - argv.extend(['-t', mount_type]) - if options: - argv.extend(['-o', options]) - if flags: - argv.extend(flags) - - if src is not None: - argv += [src] - argv += [dest] - - status, _ = utils._call( - argv, - terminate=True, - stdout=stdout, - stderr=stderr - ) - - if status != 0: - raise ElementError('`{}` failed with exit code {}' - .format(' '.join(argv), status)) - - return dest - - def _umount(self, path, stdout=sys.stdout, stderr=sys.stderr): - - cmd = [utils.get_host_tool('umount'), '-R', path] - status, _ = utils._call( - cmd, - terminate=True, - stdout=stdout, - stderr=stderr - ) - - if status != 0: - raise ElementError('`{}` failed with exit code {}' - .format(' '.join(cmd), status)) - - # mount() - # - # A wrapper for the `mount` command. The device is unmounted when - # the context is left. - # - # Args: - # src (str) - The directory to mount - # dest (str) - The directory to mount to - # mount_type (str|None) - The mount type (can be omitted or None) - # kwargs - Arguments to pass to the mount command, such as `ro=True` - # - # Yields: - # (str) The path to the destination - # - @contextmanager - def mount(self, dest, src=None, mount_type=None, - stdout=sys.stdout, stderr=sys.stderr, **kwargs): - - options = ','.join([key for key, val in kwargs.items() if val]) - - yield self._mount(dest, src, mount_type, stdout=stdout, stderr=stderr, options=options) - - self._umount(dest, stdout, stderr) - - # bind_mount() - # - # Mount a directory to a different location (a hardlink for all - # intents and purposes). The directory is unmounted when the - # context is left. - # - # Args: - # src (str) - The directory to mount - # dest (str) - The directory to mount to - # kwargs - Arguments to pass to the mount command, such as `ro=True` - # - # Yields: - # (str) The path to the destination - # - # While this is equivalent to `mount --rbind`, this option may not - # exist and can be dangerous, requiring careful cleanupIt is - # recommended to use this function over a manual mount invocation. - # - @contextmanager - def bind_mount(self, dest, src=None, stdout=sys.stdout, stderr=sys.stderr, - **kwargs): - - kwargs['rbind'] = True - options = ','.join([key for key, val in kwargs.items() if val]) - - path = self._mount(dest, src, None, stdout, stderr, options) - - # Make the rbind a slave to avoid unmounting vital devices in - # /proc - self._mount(dest, flags=['--make-rslave']) - - yield path - - self._umount(dest, stdout, stderr) - - -class SandboxChroot(Sandbox): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - self.platform = self._get_context()._platform - self.mount = Mount(self.platform) - - def run(self, command, flags, cwd=None, env=None): - - # Default settings - if cwd is None: - cwd = self._get_work_directory() - - if cwd is None: - cwd = '/' - - if env is None: - env = self._get_environment() - - # Command must be a list - if isinstance(command, str): - command = [command] - - stdout, stderr = self._get_output() - - # Create a chroot directory and run the command inside it - with ExitStack() as stack: - if flags & SandboxFlags.INTERACTIVE: - stdin = sys.stdin - status = self.run_in_interactive_sandbox(command, env, flags, cwd) - else: - stdin = stack.enter_context(open(os.devnull, 'r')) - status = self.run_in_sandbox(command, stdin, stdout, - stderr, cwd, env, flags) - - return status - - # run_in_sandbox() - # - # A helper function to pass the command to the chroot. - # - # Args: - # command (list): The command to execute in the chroot - # stdin (file): The stdin - # stdout (file): The stdout - # stderr (file): The stderr - # interactive (bool): Whether the sandbox should be run interactively - # cwd (str): The current working directory - # env (dict): The environment variables to use while executing the command - # - # Returns: - # (int): The exit code of the executed command - # - def run_in_sandbox(self, command, stdin, stdout, stderr, cwd, env, flags): - # Hack to ensure a module required for exception handling is - # not loaded in the chroot. - # - # Propagates from here: - # https://github.com/CodethinkLabs/sandboxlib/blob/7e2a551189b5ffb7a0124db63964bdec69ead3e8/sandboxlib/chroot.py#L231 - _ = "Some Text".encode('unicode-escape') - - with ExitStack() as stack: - stack.enter_context(self.create_devices(flags)) - stack.enter_context(self.mount_dirs(flags, stdout, stderr)) - - try: - code, _ = utils._call( - command, - terminate=True, - close_fds=True, - cwd=os.path.join(self.get_directory(), cwd.lstrip(os.sep)), - env=env, - stdin=stdin, - stdout=stdout, - stderr=stderr, - # If you try to put gtk dialogs here Tristan (either) - # will personally scald you - preexec_fn=lambda: (os.chroot(self.get_directory()), os.chdir(cwd)), - start_new_session=flags & SandboxFlags.INTERACTIVE - ) - except subprocess.SubprocessError as e: - # Exceptions in preexec_fn are simply reported as - # 'Exception occurred in preexec_fn', turn these into - # a more readable message. - if '{}'.format(e) == 'Exception occurred in preexec_fn.': - raise ElementError('Could not chroot into {} or chdir into {}. ' - 'Ensure you are root and that the relevant directory exists.' - .format(self.get_directory(), cwd)) from e - else: - raise ElementError('Could not run command {}: {}'.format(command, e)) from e - - if code != 0: - raise ElementError("{} failed with exit code {}".format(command, code)) - - return code - - # run_in_interactive_sandbox() - # - # Run an interactive command. - # - # - # Args: - # command (list): The command to execute in the chroot - # stdin (file): The stdin - # stdout (file): The stdout - # stderr (file): The stderr - # interactive (bool): Whether the sandbox should be run interactively - # cwd (str): The current working directory - # env (dict): The environment variables to use while executing the command - # - # Returns: - # (int): The exit code of the executed command - # - # The method is similar to SandboxChroot.run_in_sandbox(), but - # does not create a subprocess to allow the interactive session to - # communicate with the frontend. It foregoes changing into `cwd` - # since this is less important in an interactive session. - # - def run_in_interactive_sandbox(self, command, env, flags, cwd): - - with ExitStack() as stack: - stack.enter_context(self.create_devices(flags)) - stack.enter_context(self.mount_dirs(flags, sys.stdout, sys.stderr)) - - process = subprocess.Popen( - command, - cwd=os.path.join(self.get_directory(), cwd.lstrip(os.sep)), - close_fds=True, - preexec_fn=lambda: (os.chroot(self.get_directory()), os.chdir(cwd)), - env=env - ) - process.communicate() - - return process.poll() - - # create_devices() - # - # Create the nodes in /dev/ usually required for builds (null, - # none, etc.) - # - # Args: - # flags (:class:`.SandboxFlags`): The sandbox flags - # - @contextmanager - def create_devices(self, flags): - - devices = [] - # When we are interactive, we'd rather mount /dev due to the - # sheer number of devices - if not flags & SandboxFlags.INTERACTIVE: - os.makedirs('/dev/', exist_ok=True) - - for device in Sandbox.DEVICES: - location = os.path.join(self.get_directory(), device.lstrip(os.sep)) - os.makedirs(os.path.dirname(location), exist_ok=True) - try: - # If the image already contains a device, remove - # it, since the device numbers may be different on - # different systems. - os.remove(location) - - devices.append(self.mknod(device, location)) - except OSError as err: - if err.errno == 1: - raise ElementError("Permission denied while creating device node: {}.".format(err) + - "BuildStream reqiures root permissions for these setttings.") - - yield - - for device in devices: - os.remove(device) - - # mount() - # - # Mount paths required for the command. - # - # Args: - # flags (:class:`.SandboxFlags`): The sandbox flags - # - @contextmanager - def mount_dirs(self, flags, stdout, stderr): - - # FIXME: This should probably keep track of potentially - # already existing files a la _sandboxwrap.py:239 - - # To successfully mount the sysroot RO, we need to: - # - Mount / RO first since solaris can't remount a bind mount - # - Create missing directories and mount points - # - Move marked directories to a scratch directory to keep their RW state - # - Mount / RO - # - Mount marked directories RW - # - Mount system directories (/dev, /proc/, /tmp, ...) - # - # - execute command - # - # - Unmount system and marked directories - # - Unmount / - - # Create missing system directories - if flags & SandboxFlags.INTERACTIVE: - # We mount /dev in interactive sandboxes to give the - # user a working console - dev_src, dev_point = self.get_mount_location('/dev/', '/dev/') - - proc_src, proc_point = self.get_mount_location('/proc/', '/proc/') - tmp_src, tmp_point = self.get_mount_location('/tmp', '/tmp') - - marked_directories = self._get_marked_directories() - marked_locations = [] - - for mark in marked_directories: - host_location = os.path.join(self.get_directory(), - mark['directory'].lstrip(os.sep)) - scratch_location = os.path.join(self._get_scratch_directory(), - mark['directory'].lstrip(os.sep)) - - # On the first invocation, move the marked directories to - # the scratch directory to allow mounting them read-write - - # Create the host location if it does not exist yet - os.makedirs(host_location, exist_ok=True) - - # Move marked directories before mounting / RO - shutil.move(host_location, scratch_location) - - # Record the mount locations - marked_locations.append(self.get_mount_location(scratch_location, mark['directory'])) - - with ExitStack() as stack: - # / mount - root_mount = self.mount.bind_mount(self.get_directory(), self.get_directory(), - stdout, stderr, - ro=flags & SandboxFlags.ROOT_READ_ONLY) - # Since / needs to be unmounted after and before - # everything else has been unmounted, manually enter/exit - # the context. - root_mount.__enter__() - - # System mounts - if flags & SandboxFlags.INTERACTIVE: - stack.enter_context(self.mount.bind_mount(dev_point, dev_src, stdout, stderr)) - - stack.enter_context(self.mount.mount(proc_point, proc_src, 'proc', stdout, stderr, ro=True)) - stack.enter_context(self.mount.bind_mount(tmp_point, tmp_src, stdout, stderr)) - - # Mark mounts - for src, point in marked_locations: - stack.enter_context(self.mount.bind_mount(point, src, stdout, stderr)) - - yield - - root_mount.__exit__(None, None, None) - - # Move back mark mounts - for src, point in marked_locations: - shutil.rmtree(point) - shutil.move(src, point) - - # get_mount_location() - # - # Return a tuple that indicates the locations to mount a host - # directory to and from to mount it to the corresponding path in - # the chroot. - # - # Args: - # host_dir (str): The path to the host dir to mount. - # chroot_dir (str): The dir in the chroot to mount it to. - # - # Returns: - # (str, str) - (source, point) - # - def get_mount_location(self, host_dir, chroot_dir): - point = os.path.join(self.get_directory(), chroot_dir.lstrip(os.sep)) - - os.makedirs(host_dir, exist_ok=True) - os.makedirs(point, exist_ok=True) - - return (host_dir, point) - - # mknod() - # - # Create a device node equivalent to the given source node - # - # Args: - # source (str): Path of the device to mimic (e.g. '/dev/null') - # target (str): Location to create the new device in - # - # Returns: - # target (str): The location of the created node - # - def mknod(self, source, target): - try: - dev = os.stat(source) - - major = os.major(dev.st_rdev) - minor = os.minor(dev.st_rdev) - - target_dev = os.makedev(major, minor) - - os.mknod(target, mode=stat.S_IFCHR | dev.st_mode, device=target_dev) - - except PermissionError as e: - raise ElementError('Could not create device {}, ensure that you have root permissions: {}') - - except OSError as e: - raise ElementError('Could not create device {}: {}' - .format(target, e)) from e - - return target diff --git a/buildstream/sandbox/__init__.py b/buildstream/sandbox/__init__.py new file mode 100644 index 000000000..c1f38eac4 --- /dev/null +++ b/buildstream/sandbox/__init__.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Maat <tristan.maat@codethink.co.uk> + +from .sandbox import Sandbox, SandboxFlags, Mount, MountMap +from ._sandboxchroot import SandboxChroot +from ._sandboxbwrap import SandboxBwrap diff --git a/buildstream/sandbox/_mount.py b/buildstream/sandbox/_mount.py new file mode 100644 index 000000000..9b2c4cc1d --- /dev/null +++ b/buildstream/sandbox/_mount.py @@ -0,0 +1,125 @@ +import sys +from contextlib import contextmanager + +from .. import ElementError +from .. import utils, _signals + + +# A class to wrap the `mount` and `umount` commands +class Mount(object): + @classmethod + def _mount(cls, dest, src=None, mount_type=None, + stdout=sys.stdout, stderr=sys.stderr, options=None, + flags=None): + + argv = [utils.get_host_tool('mount')] + if mount_type: + argv.extend(['-t', mount_type]) + if options: + argv.extend(['-o', options]) + if flags: + argv.extend(flags) + + if src is not None: + argv += [src] + argv += [dest] + + status, _ = utils._call( + argv, + terminate=True, + stdout=stdout, + stderr=stderr + ) + + if status != 0: + raise ElementError('`{}` failed with exit code {}' + .format(' '.join(argv), status)) + + return dest + + @classmethod + def _umount(cls, path, stdout=sys.stdout, stderr=sys.stderr): + + cmd = [utils.get_host_tool('umount'), '-R', path] + status, _ = utils._call( + cmd, + terminate=True, + stdout=stdout, + stderr=stderr + ) + + if status != 0: + raise ElementError('`{}` failed with exit code {}' + .format(' '.join(cmd), status)) + + # mount() + # + # A wrapper for the `mount` command. The device is unmounted when + # the context is left. + # + # Args: + # dest (str) - The directory to mount to + # src (str) - The directory to mount + # stdout (file) - stdout + # stderr (file) - stderr + # mount_type (str|None) - The mount type (can be omitted or None) + # kwargs - Arguments to pass to the mount command, such as `ro=True` + # + # Yields: + # (str) The path to the destination + # + @classmethod + @contextmanager + def mount(cls, dest, src=None, stdout=sys.stdout, + stderr=sys.stderr, mount_type=None, **kwargs): + + def kill_proc(): + cls._umount(dest, stdout, stderr) + + options = ','.join([key for key, val in kwargs.items() if val]) + + with _signals.terminator(kill_proc): + yield cls._mount(dest, src, mount_type, stdout=stdout, stderr=stderr, options=options) + + cls._umount(dest, stdout, stderr) + + # bind_mount() + # + # Mount a directory to a different location (a hardlink for all + # intents and purposes). The directory is unmounted when the + # context is left. + # + # Args: + # dest (str) - The directory to mount to + # src (str) - The directory to mount + # stdout (file) - stdout + # stderr (file) - stderr + # kwargs - Arguments to pass to the mount command, such as `ro=True` + # + # Yields: + # (str) The path to the destination + # + # While this is equivalent to `mount --rbind`, this option may not + # exist and can be dangerous, requiring careful cleanupIt is + # recommended to use this function over a manual mount invocation. + # + @classmethod + @contextmanager + def bind_mount(cls, dest, src=None, stdout=sys.stdout, + stderr=sys.stderr, **kwargs): + + def kill_proc(): + cls._umount(dest, stdout, stderr) + + kwargs['rbind'] = True + options = ','.join([key for key, val in kwargs.items() if val]) + + path = cls._mount(dest, src, None, stdout, stderr, options) + + with _signals.terminator(kill_proc): + # Make the rbind a slave to avoid unmounting vital devices in + # /proc + cls._mount(dest, flags=['--make-rslave']) + yield path + + cls._umount(dest, stdout, stderr) diff --git a/buildstream/_sandboxbwrap.py b/buildstream/sandbox/_sandboxbwrap.py index af9cd51d6..2ac17b825 100644 --- a/buildstream/_sandboxbwrap.py +++ b/buildstream/sandbox/_sandboxbwrap.py @@ -20,125 +20,16 @@ # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> import os import sys -import subprocess -import shutil -import re -import tempfile -import psutil -import signal -import errno import time -from collections import OrderedDict -from contextlib import contextmanager, ExitStack - -from . import utils, _signals -from . import Sandbox, SandboxFlags -from ._fuse import SafeHardlinks - - -# Mount() -# -# Helper data object representing a single mount point in the mount map -# -class Mount(): - def __init__(self, sandbox, mount_point, safe_hardlinks): - scratch_directory = sandbox._get_scratch_directory() - root_directory = sandbox.get_directory() - - self.mount_point = mount_point - self.safe_hardlinks = safe_hardlinks - - # FIXME: When the criteria for mounting something and it's parent - # mount is identical, then there is no need to mount an additional - # fuse layer (i.e. if the root is read-write and there is a directory - # marked for staged artifacts directly within the rootfs, they can - # safely share the same fuse layer). - # - # In these cases it would be saner to redirect the sub-mount to - # a regular mount point within the parent's redirected mount. - # - if self.safe_hardlinks: - # Redirected mount - self.mount_origin = os.path.join(root_directory, mount_point.lstrip(os.sep)) - self.mount_base = os.path.join(scratch_directory, utils.url_directory_name(mount_point)) - self.mount_source = os.path.join(self.mount_base, 'mount') - self.mount_tempdir = os.path.join(self.mount_base, 'temp') - os.makedirs(self.mount_origin, exist_ok=True) - os.makedirs(self.mount_source, exist_ok=True) - os.makedirs(self.mount_tempdir, exist_ok=True) - else: - # No redirection needed - self.mount_source = os.path.join(root_directory, mount_point.lstrip(os.sep)) - os.makedirs(self.mount_source, exist_ok=True) - - @contextmanager - def mounted(self, sandbox): - if self.safe_hardlinks: - mount = SafeHardlinks(self.mount_origin, self.mount_tempdir) - with mount.mounted(self.mount_source): - yield - else: - # Nothing to mount here - yield - - -# MountMap() -# -# Helper object for mapping of the sandbox mountpoints -# -# Args: -# sandbox (Sandbox): The sandbox object -# root_readonly (bool): Whether the sandbox root is readonly -# marks (list): List of dictionaries returned by Sandbox._get_marked_directories() -# -class MountMap(): +import errno +import signal +import subprocess +from contextlib import ExitStack - def __init__(self, sandbox, root_readonly): - # We will be doing the mounts in the order in which they were declared. - self.mounts = OrderedDict() +import psutil - # We want safe hardlinks on rootfs whenever root is not readonly - if root_readonly: - self.mounts['/'] = Mount(sandbox, '/', False) - else: - self.mounts['/'] = Mount(sandbox, '/', True) - - for mark in sandbox._get_marked_directories(): - directory = mark['directory'] - artifact = mark['artifact'] - - # We want safe hardlinks for any non-root directory where - # artifacts will be staged to - self.mounts[directory] = Mount(sandbox, directory, artifact) - - # get_mount_source() - # - # Gets the host directory where the mountpoint in the - # sandbox should be bind mounted from - # - # Args: - # mountpoint (str): The absolute mountpoint path inside the sandbox - # - # Returns: - # The host path to be mounted at the mount point - # - def get_mount_source(self, mountpoint): - return self.mounts[mountpoint].mount_source - - # mounted() - # - # A context manager which ensures all the mount sources - # were mounted with any fuse layers which may have been needed. - # - # Args: - # sandbox (Sandbox): The sandbox - # - @contextmanager - def mounted(self, sandbox): - with ExitStack() as stack: - for mountpoint, mount in self.mounts.items(): - stack.enter_context(mount.mounted(sandbox)) - yield +from .. import utils, _signals +from . import Sandbox, SandboxFlags, MountMap # SandboxBwrap() diff --git a/buildstream/sandbox/_sandboxchroot.py b/buildstream/sandbox/_sandboxchroot.py new file mode 100644 index 000000000..7131200b1 --- /dev/null +++ b/buildstream/sandbox/_sandboxchroot.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Maat <tristan.maat@codethink.co.uk> +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +import os +import sys +import stat +import subprocess +from contextlib import contextmanager, ExitStack + +from .. import ElementError +from .. import utils +from ._mount import Mount +from . import Sandbox, SandboxFlags, MountMap + + +class SandboxChroot(Sandbox): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.platform = self._get_context()._platform + self.mount_map = None + + def run(self, command, flags, cwd=None, env=None): + + # Default settings + if cwd is None: + cwd = self._get_work_directory() + + if cwd is None: + cwd = '/' + + if env is None: + env = self._get_environment() + + # Command must be a list + if isinstance(command, str): + command = [command] + + stdout, stderr = self._get_output() + + # Create the mount map, this will tell us where + # each mount point needs to be mounted from and to + self.mount_map = MountMap(self, flags & SandboxFlags.ROOT_READ_ONLY) + + # Create a sysroot and run the command inside it + with ExitStack() as stack: + os.makedirs('/var/run/buildstream', exist_ok=True) + + # FIXME: While we do not currently do anything to prevent + # network access, we also don't copy /etc/resolv.conf to + # the new rootfs. + # + # This effectively disables network access, since DNs will + # never resolve, so anything a normal process wants to do + # will fail. Malicious processes could gain rights to + # anything anyway. + # + # Nonetheless a better solution could perhaps be found. + + rootfs = stack.enter_context(utils._tempdir(dir='/var/run/buildstream')) + stack.enter_context(self.create_devices(self.get_directory(), flags)) + stack.enter_context(self.mount_dirs(rootfs, flags, stdout, stderr)) + + if flags & SandboxFlags.INTERACTIVE: + stdin = sys.stdin + else: + stdin = stack.enter_context(open(os.devnull, 'r')) + + status = self.chroot(rootfs, command, stdin, stdout, + stderr, cwd, env, flags) + + return status + + # chroot() + # + # A helper function to chroot into the rootfs. + # + # Args: + # rootfs (str): The path of the sysroot to chroot into + # command (list): The command to execute in the chroot env + # stdin (file): The stdin + # stdout (file): The stdout + # stderr (file): The stderr + # cwd (str): The current working directory + # env (dict): The environment variables to use while executing the command + # flags (:class:`SandboxFlags`): The flags to enable on the sandbox + # + # Returns: + # (int): The exit code of the executed command + # + def chroot(self, rootfs, command, stdin, stdout, stderr, cwd, env, flags): + + try: + code, _ = utils._call( + command, + terminate=True, + close_fds=True, + cwd=os.path.join(rootfs, cwd.lstrip(os.sep)), + env=env, + stdin=stdin, + stdout=stdout, + stderr=stderr, + # If you try to put gtk dialogs here Tristan (either) + # will personally scald you + preexec_fn=lambda: (os.chroot(rootfs), os.chdir(cwd)), + start_new_session=flags & SandboxFlags.INTERACTIVE + ) + except subprocess.SubprocessError as e: + # Exceptions in preexec_fn are simply reported as + # 'Exception occurred in preexec_fn', turn these into + # a more readable message. + if '{}'.format(e) == 'Exception occurred in preexec_fn.': + raise ElementError('Could not chroot into {} or chdir into {}. ' + 'Ensure you are root and that the relevant directory exists.' + .format(rootfs, cwd)) from e + else: + raise ElementError('Could not run command {}: {}'.format(command, e)) from e + + if code != 0: + raise ElementError("{} failed with exit code {}".format(command, code)) + + return code + + # create_devices() + # + # Create the nodes in /dev/ usually required for builds (null, + # none, etc.) + # + # Args: + # rootfs (str): The path of the sysroot to prepare + # flags (:class:`.SandboxFlags`): The sandbox flags + # + @contextmanager + def create_devices(self, rootfs, flags): + + devices = [] + # When we are interactive, we'd rather mount /dev due to the + # sheer number of devices + if not flags & SandboxFlags.INTERACTIVE: + + for device in Sandbox.DEVICES: + location = os.path.join(rootfs, device.lstrip(os.sep)) + os.makedirs(os.path.dirname(location), exist_ok=True) + try: + if os.path.exists(location): + os.remove(location) + + devices.append(self.mknod(device, location)) + except OSError as err: + if err.errno == 1: + raise ElementError("Permission denied while creating device node: {}.".format(err) + + "BuildStream reqiures root permissions for these setttings.") + else: + raise + + yield + + for device in devices: + os.remove(device) + + # mount_dirs() + # + # Mount paths required for the command. + # + # Args: + # rootfs (str): The path of the sysroot to prepare + # flags (:class:`.SandboxFlags`): The sandbox flags + # stdout (file): The stdout + # stderr (file): The stderr + # + @contextmanager + def mount_dirs(self, rootfs, flags, stdout, stderr): + + # FIXME: This should probably keep track of potentially + # already existing files a la _sandboxwrap.py:239 + + @contextmanager + def mount_point(point, **kwargs): + mount_source = self.mount_map.get_mount_source(point) + mount_point = os.path.join(rootfs, point.lstrip(os.sep)) + + with Mount.bind_mount(mount_point, src=mount_source, stdout=stdout, stderr=stderr, **kwargs): + yield + + @contextmanager + def mount_src(src, **kwargs): + mount_point = os.path.join(rootfs, src.lstrip(os.sep)) + os.makedirs(mount_point, exist_ok=True) + + with Mount.bind_mount(mount_point, src=src, stdout=stdout, stderr=stderr, **kwargs): + yield + + with ExitStack() as stack: + stack.enter_context(self.mount_map.mounted(self)) + + stack.enter_context(mount_point('/')) + + if flags & SandboxFlags.INTERACTIVE: + stack.enter_context(mount_src('/dev')) + + stack.enter_context(mount_src('/tmp')) + stack.enter_context(mount_src('/proc')) + + for mark in self._get_marked_directories(): + stack.enter_context(mount_point(mark['directory'])) + + # Remount root RO if necessary + if flags & flags & SandboxFlags.ROOT_READ_ONLY: + root_mount = Mount.mount(rootfs, stdout=stdout, stderr=stderr, remount=True, ro=True, bind=True) + # Since the exit stack has already registered a mount + # for this path, we do not need to register another + # umount call. + root_mount.__enter__() + + yield + + # mknod() + # + # Create a device node equivalent to the given source node + # + # Args: + # source (str): Path of the device to mimic (e.g. '/dev/null') + # target (str): Location to create the new device in + # + # Returns: + # target (str): The location of the created node + # + def mknod(self, source, target): + try: + dev = os.stat(source) + major = os.major(dev.st_rdev) + minor = os.minor(dev.st_rdev) + + target_dev = os.makedev(major, minor) + + os.mknod(target, mode=stat.S_IFCHR | dev.st_mode, device=target_dev) + + except PermissionError as e: + raise ElementError('Could not create device {}, ensure that you have root permissions: {}') + + except OSError as e: + raise ElementError('Could not create device {}: {}' + .format(target, e)) from e + + return target diff --git a/buildstream/sandbox.py b/buildstream/sandbox/sandbox.py index 4462f4424..fe69eae94 100644 --- a/buildstream/sandbox.py +++ b/buildstream/sandbox/sandbox.py @@ -29,8 +29,12 @@ conform to this interface. """ import os +from collections import OrderedDict +from contextlib import contextmanager, ExitStack -from . import ImplError +from .. import utils +from .. import ImplError +from .._fuse import SafeHardlinks class SandboxFlags(): @@ -61,12 +65,123 @@ class SandboxFlags(): """ +# Mount() +# +# Helper data object representing a single mount point in the mount map +# +class Mount(): + def __init__(self, sandbox, mount_point, safe_hardlinks): + scratch_directory = sandbox._get_scratch_directory() + root_directory = sandbox.get_directory() + + self.mount_point = mount_point + self.safe_hardlinks = safe_hardlinks + + # FIXME: When the criteria for mounting something and it's parent + # mount is identical, then there is no need to mount an additional + # fuse layer (i.e. if the root is read-write and there is a directory + # marked for staged artifacts directly within the rootfs, they can + # safely share the same fuse layer). + # + # In these cases it would be saner to redirect the sub-mount to + # a regular mount point within the parent's redirected mount. + # + if self.safe_hardlinks: + # Redirected mount + self.mount_origin = os.path.join(root_directory, mount_point.lstrip(os.sep)) + self.mount_base = os.path.join(scratch_directory, utils.url_directory_name(mount_point)) + self.mount_source = os.path.join(self.mount_base, 'mount') + self.mount_tempdir = os.path.join(self.mount_base, 'temp') + os.makedirs(self.mount_origin, exist_ok=True) + os.makedirs(self.mount_source, exist_ok=True) + os.makedirs(self.mount_tempdir, exist_ok=True) + else: + # No redirection needed + self.mount_source = os.path.join(root_directory, mount_point.lstrip(os.sep)) + os.makedirs(self.mount_source, exist_ok=True) + + @contextmanager + def mounted(self, sandbox): + if self.safe_hardlinks: + mount = SafeHardlinks(self.mount_origin, self.mount_tempdir) + with mount.mounted(self.mount_source): + yield + else: + # Nothing to mount here + yield + + +# MountMap() +# +# Helper object for mapping of the sandbox mountpoints +# +# Args: +# sandbox (Sandbox): The sandbox object +# root_readonly (bool): Whether the sandbox root is readonly +# +class MountMap(): + + def __init__(self, sandbox, root_readonly): + # We will be doing the mounts in the order in which they were declared. + self.mounts = OrderedDict() + + # We want safe hardlinks on rootfs whenever root is not readonly + self.mounts['/'] = Mount(sandbox, '/', not root_readonly) + + for mark in sandbox._get_marked_directories(): + directory = mark['directory'] + artifact = mark['artifact'] + + # We want safe hardlinks for any non-root directory where + # artifacts will be staged to + self.mounts[directory] = Mount(sandbox, directory, artifact) + + # get_mount_source() + # + # Gets the host directory where the mountpoint in the + # sandbox should be bind mounted from + # + # Args: + # mountpoint (str): The absolute mountpoint path inside the sandbox + # + # Returns: + # The host path to be mounted at the mount point + # + def get_mount_source(self, mountpoint): + return self.mounts[mountpoint].mount_source + + # mounted() + # + # A context manager which ensures all the mount sources + # were mounted with any fuse layers which may have been needed. + # + # Args: + # sandbox (Sandbox): The sandbox + # + @contextmanager + def mounted(self, sandbox): + with ExitStack() as stack: + for mountpoint, mount in self.mounts.items(): + stack.enter_context(mount.mounted(sandbox)) + yield + + class Sandbox(): + + # Minimal set of devices for the sandbox + DEVICES = [ + '/dev/urandom', + '/dev/random', + '/dev/zero', + '/dev/null' + ] + """Sandbox() Sandbox programming interface for :class:`.Element` plugins. """ - def __init__(self, context, project, directory, stdout=None, stderr=None): + def __init__(self, context, project, directory, + stdout=None, stderr=None): self.__context = context self.__project = project self.__stdout = stdout |