diff options
author | William Salmon <will.salmon@codethink.co.uk> | 2019-06-05 14:22:34 +0100 |
---|---|---|
committer | bst-marge-bot <marge-bot@buildstream.build> | 2019-07-12 06:57:52 +0000 |
commit | 33272aa7764c03f7d0b3a7b36f08636f883c3e69 (patch) | |
tree | 675034c2720fdf2fd80a3d6da29b77fab15e414e | |
parent | 24426ebe31fc2ad297b352a1d332c7cf158ef5c2 (diff) | |
download | buildstream-33272aa7764c03f7d0b3a7b36f08636f883c3e69.tar.gz |
Refactor of Platform and Sandbox
-rw-r--r-- | .gitlab-ci.yml | 2 | ||||
-rwxr-xr-x | setup.py | 6 | ||||
-rw-r--r-- | src/buildstream/_exceptions.py | 4 | ||||
-rw-r--r-- | src/buildstream/_platform/darwin.py | 24 | ||||
-rw-r--r-- | src/buildstream/_platform/fallback.py | 37 | ||||
-rw-r--r-- | src/buildstream/_platform/linux.py | 172 | ||||
-rw-r--r-- | src/buildstream/_platform/platform.py | 71 | ||||
-rw-r--r-- | src/buildstream/_platform/unix.py | 56 | ||||
-rw-r--r-- | src/buildstream/sandbox/_sandboxbwrap.py | 89 | ||||
-rw-r--r-- | src/buildstream/sandbox/_sandboxchroot.py | 30 | ||||
-rw-r--r-- | src/buildstream/sandbox/sandbox.py | 1 | ||||
-rw-r--r-- | src/buildstream/testing/_utils/site.py | 22 | ||||
-rw-r--r-- | src/buildstream/testing/runcli.py | 5 | ||||
-rw-r--r-- | tests/integration/build-uid.py | 8 | ||||
-rw-r--r-- | tests/integration/cachedfail.py | 24 | ||||
-rw-r--r-- | tests/integration/sandbox-bwrap.py | 8 | ||||
-rw-r--r-- | tests/sandboxes/missing_dependencies.py | 8 | ||||
-rw-r--r-- | tox.ini | 1 |
18 files changed, 354 insertions, 214 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 93a8b318b..f4b174a7d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -87,7 +87,7 @@ tests-unix: image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-47052095 <<: *tests variables: - BST_FORCE_BACKEND: "unix" + BST_FORCE_SANDBOX: "chroot" script: @@ -84,8 +84,12 @@ def bwrap_too_old(major, minor, patch): def check_for_bwrap(): - platform = os.environ.get('BST_FORCE_BACKEND', '') or sys.platform + platform = sys.platform + if platform.startswith('linux'): + sandbox = os.environ.get('BST_FORCE_SANDBOX', "bwrap") + if sandbox != 'bwrap': + return bwrap_path = shutil.which('bwrap') if not bwrap_path: warn_bwrap("Bubblewrap not found") diff --git a/src/buildstream/_exceptions.py b/src/buildstream/_exceptions.py index 82f1fe8ed..034a5125a 100644 --- a/src/buildstream/_exceptions.py +++ b/src/buildstream/_exceptions.py @@ -256,8 +256,8 @@ class ImplError(BstError): # # Raised if the current platform is not supported. class PlatformError(BstError): - def __init__(self, message, reason=None): - super().__init__(message, domain=ErrorDomain.PLATFORM, reason=reason) + def __init__(self, message, reason=None, detail=None): + super().__init__(message, domain=ErrorDomain.PLATFORM, reason=reason, detail=detail) # SandboxError diff --git a/src/buildstream/_platform/darwin.py b/src/buildstream/_platform/darwin.py index 282a5b445..e8c1ffaf3 100644 --- a/src/buildstream/_platform/darwin.py +++ b/src/buildstream/_platform/darwin.py @@ -28,16 +28,6 @@ class Darwin(Platform): # This value comes from OPEN_MAX in syslimits.h OPEN_MAX = 10240 - def create_sandbox(self, *args, **kwargs): - kwargs['dummy_reason'] = \ - "OSXFUSE is not supported and there are no supported sandbox " + \ - "technologies for MacOS at this time" - return SandboxDummy(*args, **kwargs) - - def check_sandbox_config(self, config): - # Accept all sandbox configs as it's irrelevant with the dummy sandbox (no Sandbox.run). - return True - def get_cpu_count(self, cap=None): cpu_count = os.cpu_count() if cap is None: @@ -62,3 +52,17 @@ class Darwin(Platform): old_soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) soft_limit = min(max(self.OPEN_MAX, old_soft_limit), hard_limit) resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit)) + + def _setup_dummy_sandbox(self): + def _check_dummy_sandbox_config(config): + return True + self.check_sandbox_config = _check_dummy_sandbox_config + + def _create_dummy_sandbox(*args, **kwargs): + kwargs['dummy_reason'] = \ + "OSXFUSE is not supported and there are no supported sandbox " + \ + "technologies for MacOS at this time" + return SandboxDummy(*args, **kwargs) + self.create_sandbox = _create_dummy_sandbox + + return True diff --git a/src/buildstream/_platform/fallback.py b/src/buildstream/_platform/fallback.py new file mode 100644 index 000000000..39669e0c2 --- /dev/null +++ b/src/buildstream/_platform/fallback.py @@ -0,0 +1,37 @@ +# +# Copyright (C) 2018 Bloomberg Finance LP +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +from ..sandbox import SandboxDummy + +from .platform import Platform + + +class Fallback(Platform): + + def _setup_dummy_sandbox(self): + def _check_dummy_sandbox_config(config): + return True + self.check_sandbox_config = _check_dummy_sandbox_config + + def _create_dummy_sandbox(*args, **kwargs): + kwargs['dummy_reason'] = \ + ("FallBack platform only implements dummy sandbox, " + "Buildstream may be having issues correctly detecting your platform, platform " + "can be forced with BST_FORCE_BACKEND") + return SandboxDummy(*args, **kwargs) + self.create_sandbox = _create_dummy_sandbox + + return True diff --git a/src/buildstream/_platform/linux.py b/src/buildstream/_platform/linux.py index e4ce02572..3d85fdf34 100644 --- a/src/buildstream/_platform/linux.py +++ b/src/buildstream/_platform/linux.py @@ -1,5 +1,6 @@ # # Copyright (C) 2017 Codethink Limited +# Copyright (C) 2018 Bloomberg Finance LP # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -18,133 +19,100 @@ # Tristan Maat <tristan.maat@codethink.co.uk> import os -import subprocess -from .. import _site from .. import utils from ..sandbox import SandboxDummy from .platform import Platform -from .._exceptions import PlatformError class Linux(Platform): - def __init__(self): + def _setup_sandbox(self, force_sandbox): + sandbox_setups = { + 'bwrap': self._setup_bwrap_sandbox, + 'chroot': self._setup_chroot_sandbox, + 'dummy': self._setup_dummy_sandbox, + } - super().__init__() + preferred_sandboxes = [ + 'bwrap', + ] - self._uid = os.geteuid() - self._gid = os.getegid() - - self._have_fuse = os.path.exists("/dev/fuse") - - bwrap_version = _site.get_bwrap_version() + self._try_sandboxes(force_sandbox, sandbox_setups, preferred_sandboxes) - if bwrap_version is None: - self._bwrap_exists = False - self._have_good_bwrap = False - self._die_with_parent_available = False - self._json_status_available = False - else: - self._bwrap_exists = True - self._have_good_bwrap = (0, 1, 2) <= bwrap_version - self._die_with_parent_available = (0, 1, 8) <= bwrap_version - self._json_status_available = (0, 3, 2) <= bwrap_version + def __init__(self, force_sandbox=None): + super().__init__(force_sandbox=force_sandbox) - self._local_sandbox_available = self._have_fuse and self._have_good_bwrap - - if self._local_sandbox_available: - self._user_ns_available = self._check_user_ns_available() - else: - self._user_ns_available = False + self._uid = os.geteuid() + self._gid = os.getegid() # Set linux32 option - self._linux32 = False + self.linux32 = None - def create_sandbox(self, *args, **kwargs): - if not self._local_sandbox_available: - return self._create_dummy_sandbox(*args, **kwargs) - else: - return self._create_bwrap_sandbox(*args, **kwargs) - - def check_sandbox_config(self, config): - if not self._local_sandbox_available: - # Accept all sandbox configs as it's irrelevant with the dummy sandbox (no Sandbox.run). - return True - - if self._user_ns_available: - # User namespace support allows arbitrary build UID/GID settings. - pass - elif (config.build_uid != self._uid or config.build_gid != self._gid): - # Without user namespace support, the UID/GID in the sandbox - # will match the host UID/GID. - return False - - # We can't do builds for another host or architecture except x86-32 on - # x86-64 - host_os = self.get_host_os() + def can_crossbuild(self, config): host_arch = self.get_host_arch() - if config.build_os != host_os: - raise PlatformError("Configured and host OS don't match.") - elif config.build_arch != host_arch: - # We can use linux32 for building 32bit on 64bit machines - if (host_os == "Linux" and - ((config.build_arch == "x86-32" and host_arch == "x86-64") or - (config.build_arch == "aarch32" and host_arch == "aarch64"))): - # check linux32 is available + if ((config.build_arch == "x86-32" and host_arch == "x86-64") or + (config.build_arch == "aarch32" and host_arch == "aarch64")): + if self.linux32 is None: try: utils.get_host_tool('linux32') - self._linux32 = True + self.linux32 = True except utils.ProgramNotFoundError: - pass - else: - raise PlatformError("Configured architecture and host architecture don't match.") - - return True + self.linux32 = False + return self.linux32 + return False ################################################ # Private Methods # ################################################ - def _create_dummy_sandbox(self, *args, **kwargs): - reasons = [] - if not self._have_fuse: - reasons.append("FUSE is unavailable") - if not self._have_good_bwrap: - if self._bwrap_exists: - reasons.append("`bwrap` is too old (bst needs at least 0.1.2)") - else: - reasons.append("`bwrap` executable not found") + def _setup_dummy_sandbox(self): + dummy_reasons = " and ".join(self.dummy_reasons) + + def _check_dummy_sandbox_config(config): + return True + self.check_sandbox_config = _check_dummy_sandbox_config + + def _create_dummy_sandbox(*args, **kwargs): + kwargs['dummy_reason'] = dummy_reasons + return SandboxDummy(*args, **kwargs) + self.create_sandbox = _create_dummy_sandbox - kwargs['dummy_reason'] = " and ".join(reasons) - return SandboxDummy(*args, **kwargs) + return True - def _create_bwrap_sandbox(self, *args, **kwargs): + def _setup_bwrap_sandbox(self): from ..sandbox._sandboxbwrap import SandboxBwrap - # Inform the bubblewrap sandbox as to whether it can use user namespaces or not - kwargs['user_ns_available'] = self._user_ns_available - kwargs['die_with_parent_available'] = self._die_with_parent_available - kwargs['json_status_available'] = self._json_status_available - kwargs['linux32'] = self._linux32 - return SandboxBwrap(*args, **kwargs) - - def _check_user_ns_available(self): - # Here, lets check if bwrap is able to create user namespaces, - # issue a warning if it's not available, and save the state - # locally so that we can inform the sandbox to not try it - # later on. - bwrap = utils.get_host_tool('bwrap') - whoami = utils.get_host_tool('whoami') - try: - output = subprocess.check_output([ - bwrap, - '--ro-bind', '/', '/', - '--unshare-user', - '--uid', '0', '--gid', '0', - whoami, - ], universal_newlines=True).strip() - except subprocess.CalledProcessError: - output = '' - - return output == 'root' + + # This function should only be called once. + # but if it does eg, in the tests we want to + # reset the sandbox checks + + SandboxBwrap._have_good_bwrap = None + self._check_sandbox(SandboxBwrap) + + def _check_sandbox_config_bwrap(config): + return SandboxBwrap.check_sandbox_config(self, config) + self.check_sandbox_config = _check_sandbox_config_bwrap + + def _create_bwrap_sandbox(*args, **kwargs): + kwargs['linux32'] = self.linux32 + return SandboxBwrap(*args, **kwargs) + self.create_sandbox = _create_bwrap_sandbox + + return True + + def _setup_chroot_sandbox(self): + from ..sandbox._sandboxchroot import SandboxChroot + + self._check_sandbox(SandboxChroot) + + def _check_sandbox_config_chroot(config): + return SandboxChroot.check_sandbox_config(self, config) + self.check_sandbox_config = _check_sandbox_config_chroot + + def _create_chroot_sandbox(*args, **kwargs): + return SandboxChroot(*args, **kwargs) + self.create_sandbox = _create_chroot_sandbox + + return True diff --git a/src/buildstream/_platform/platform.py b/src/buildstream/_platform/platform.py index 5f2b7081a..dab6049ea 100644 --- a/src/buildstream/_platform/platform.py +++ b/src/buildstream/_platform/platform.py @@ -1,5 +1,6 @@ # # Copyright (C) 2017 Codethink Limited +# Copyright (C) 2018 Bloomberg Finance LP # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -23,7 +24,8 @@ import sys import psutil -from .._exceptions import PlatformError, ImplError +from .._exceptions import PlatformError, ImplError, SandboxError +from .. import utils class Platform(): @@ -34,33 +36,82 @@ class Platform(): # A class to manage platform-specific details. Currently holds the # sandbox factory as well as platform helpers. # - def __init__(self): + # Args: + # force_sandbox (bool): Force bst to use a particular sandbox + # + def __init__(self, force_sandbox=None): self.maximize_open_file_limit() + self._local_sandbox = None + self.dummy_reasons = [] + self._setup_sandbox(force_sandbox) + + def _setup_sandbox(self, force_sandbox): + sandbox_setups = {'dummy': self._setup_dummy_sandbox} + preferred_sandboxes = [] + self._try_sandboxes(force_sandbox, sandbox_setups, preferred_sandboxes) + + def _try_sandboxes(self, force_sandbox, sandbox_setups, preferred_sandboxes): + # Any sandbox from sandbox_setups can be forced by BST_FORCE_SANDBOX + # But if a specific sandbox is not forced then only `first class` sandbox are tried before + # falling back to the dummy sandbox. + # Where `first_class` sandboxes are those in preferred_sandboxes + if force_sandbox: + try: + sandbox_setups[force_sandbox]() + except KeyError: + raise PlatformError("Forced Sandbox is unavailable on this platform: BST_FORCE_SANDBOX" + " is set to {} but it is not available".format(force_sandbox)) + except SandboxError as Error: + raise PlatformError("Forced Sandbox Error: BST_FORCE_SANDBOX" + " is set to {} but cannot be setup".format(force_sandbox), + detail=" and ".join(self.dummy_reasons)) from Error + else: + for good_sandbox in preferred_sandboxes: + try: + sandbox_setups[good_sandbox]() + return + except SandboxError: + continue + except utils.ProgramNotFoundError: + continue + sandbox_setups['dummy']() + + def _check_sandbox(self, Sandbox): + try: + Sandbox.check_available() + except SandboxError as Error: + self.dummy_reasons += Sandbox._dummy_reasons + raise Error @classmethod def _create_instance(cls): # Meant for testing purposes and therefore hidden in the # deepest corners of the source code. Try not to abuse this, # please? + if os.getenv('BST_FORCE_SANDBOX'): + force_sandbox = os.getenv('BST_FORCE_SANDBOX') + else: + force_sandbox = None + if os.getenv('BST_FORCE_BACKEND'): backend = os.getenv('BST_FORCE_BACKEND') - elif sys.platform.startswith('linux'): - backend = 'linux' elif sys.platform.startswith('darwin'): backend = 'darwin' + elif sys.platform.startswith('linux'): + backend = 'linux' else: - backend = 'unix' + backend = 'fallback' if backend == 'linux': from .linux import Linux as PlatformImpl # pylint: disable=cyclic-import elif backend == 'darwin': from .darwin import Darwin as PlatformImpl # pylint: disable=cyclic-import - elif backend == 'unix': - from .unix import Unix as PlatformImpl # pylint: disable=cyclic-import + elif backend == 'fallback': + from .fallback import Fallback as PlatformImpl # pylint: disable=cyclic-import else: raise PlatformError("No such platform: '{}'".format(backend)) - cls._instance = PlatformImpl() + cls._instance = PlatformImpl(force_sandbox=force_sandbox) @classmethod def get_platform(cls): @@ -167,3 +218,7 @@ class Platform(): soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) if soft_limit != hard_limit: resource.setrlimit(resource.RLIMIT_NOFILE, (hard_limit, hard_limit)) + + def _setup_dummy_sandbox(self): + raise ImplError("Platform {platform} does not implement _setup_dummy_sandbox()" + .format(platform=type(self).__name__)) diff --git a/src/buildstream/_platform/unix.py b/src/buildstream/_platform/unix.py deleted file mode 100644 index d04b0712c..000000000 --- a/src/buildstream/_platform/unix.py +++ /dev/null @@ -1,56 +0,0 @@ -# -# Copyright (C) 2017 Codethink Limited -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see <http://www.gnu.org/licenses/>. -# -# Authors: -# Tristan Maat <tristan.maat@codethink.co.uk> - -import os - -from .._exceptions import PlatformError - -from .platform import Platform - - -class Unix(Platform): - - def __init__(self): - - super().__init__() - - self._uid = os.geteuid() - self._gid = os.getegid() - - # Not necessarily 100% reliable, but we want to fail early. - if self._uid != 0: - raise PlatformError("Root privileges are required to run without bubblewrap.") - - def create_sandbox(self, *args, **kwargs): - from ..sandbox._sandboxchroot import SandboxChroot - return SandboxChroot(*args, **kwargs) - - def check_sandbox_config(self, config): - # With the chroot sandbox, the UID/GID in the sandbox - # will match the host UID/GID (typically 0/0). - if config.build_uid != self._uid or config.build_gid != self._gid: - return False - - # Check host os and architecture match - if config.build_os != self.get_host_os(): - raise PlatformError("Configured and host OS don't match.") - elif config.build_arch != self.get_host_arch(): - raise PlatformError("Configured and host architecture don't match.") - - return True diff --git a/src/buildstream/sandbox/_sandboxbwrap.py b/src/buildstream/sandbox/_sandboxbwrap.py index d2abc33d0..17f999ac0 100644 --- a/src/buildstream/sandbox/_sandboxbwrap.py +++ b/src/buildstream/sandbox/_sandboxbwrap.py @@ -1,5 +1,6 @@ # # Copyright (C) 2016 Codethink Limited +# Copyright (C) 2019 Bloomberg Finance LP # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -17,6 +18,8 @@ # Authors: # Andrew Leeming <andrew.leeming@codethink.co.uk> # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> +# William Salmon <will.salmon@codethink.co.uk> + import collections import json import os @@ -35,6 +38,7 @@ from .._exceptions import SandboxError from .. import utils, _signals from ._mount import MountMap from . import Sandbox, SandboxFlags +from .. import _site # SandboxBwrap() @@ -42,6 +46,7 @@ from . import Sandbox, SandboxFlags # Default bubblewrap based sandbox implementation. # class SandboxBwrap(Sandbox): + _have_good_bwrap = None # Minimal set of devices for the sandbox DEVICES = [ @@ -54,11 +59,83 @@ class SandboxBwrap(Sandbox): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.user_ns_available = kwargs['user_ns_available'] - self.die_with_parent_available = kwargs['die_with_parent_available'] - self.json_status_available = kwargs['json_status_available'] self.linux32 = kwargs['linux32'] + @classmethod + def check_available(cls): + cls._have_fuse = os.path.exists("/dev/fuse") + if not cls._have_fuse: + cls._dummy_reasons += ['Fuse is unavailable'] + + try: + utils.get_host_tool('bwrap') + except utils.ProgramNotFoundError as Error: + cls._bwrap_exists = False + cls._have_good_bwrap = False + cls._die_with_parent_available = False + cls._json_status_available = False + cls._dummy_reasons += ['Bubblewrap not found'] + raise SandboxError(" and ".join(cls._dummy_reasons), + reason="unavailable-local-sandbox") from Error + + bwrap_version = _site.get_bwrap_version() + + cls._bwrap_exists = True + cls._have_good_bwrap = (0, 1, 2) <= bwrap_version + cls._die_with_parent_available = (0, 1, 8) <= bwrap_version + cls._json_status_available = (0, 3, 2) <= bwrap_version + if not cls._have_good_bwrap: + cls._dummy_reasons += ['Bubblewrap is too old'] + raise SandboxError(" and ".join(cls._dummy_reasons)) + + cls._uid = os.geteuid() + cls._gid = os.getegid() + + cls.user_ns_available = cls._check_user_ns_available() + + @staticmethod + def _check_user_ns_available(): + # Here, lets check if bwrap is able to create user namespaces, + # issue a warning if it's not available, and save the state + # locally so that we can inform the sandbox to not try it + # later on. + bwrap = utils.get_host_tool('bwrap') + try: + whoami = utils.get_host_tool('whoami') + output = subprocess.check_output([ + bwrap, + '--ro-bind', '/', '/', + '--unshare-user', + '--uid', '0', '--gid', '0', + whoami, + ], universal_newlines=True).strip() + except subprocess.CalledProcessError: + output = '' + except utils.ProgramNotFoundError: + output = '' + + return output == 'root' + + @classmethod + def check_sandbox_config(cls, local_platform, config): + if cls.user_ns_available: + # User namespace support allows arbitrary build UID/GID settings. + pass + elif (config.build_uid != local_platform._uid or config.build_gid != local_platform._gid): + # Without user namespace support, the UID/GID in the sandbox + # will match the host UID/GID. + return False + + host_os = local_platform.get_host_os() + host_arch = local_platform.get_host_arch() + if config.build_os != host_os: + raise SandboxError("Configured and host OS don't match.") + elif config.build_arch != host_arch: + if not local_platform.can_crossbuild(config): + raise SandboxError("Configured architecture and host architecture don't match.") + + return True + def _run(self, command, flags, *, cwd, env): stdout, stderr = self._get_output() @@ -94,7 +171,7 @@ class SandboxBwrap(Sandbox): bwrap_command += ['--unshare-pid'] # Ensure subprocesses are cleaned up when the bwrap parent dies. - if self.die_with_parent_available: + if self._die_with_parent_available: bwrap_command += ['--die-with-parent'] # Add in the root filesystem stuff first. @@ -164,7 +241,7 @@ class SandboxBwrap(Sandbox): with ExitStack() as stack: pass_fds = () # Improve error reporting with json-status if available - if self.json_status_available: + if self._json_status_available: json_status_file = stack.enter_context(TemporaryFile()) pass_fds = (json_status_file.fileno(),) bwrap_command += ['--json-status-fd', str(json_status_file.fileno())] @@ -246,7 +323,7 @@ class SandboxBwrap(Sandbox): # a bug, bwrap mounted a tempfs here and when it exits, that better be empty. pass - if self.json_status_available: + if self._json_status_available: json_status_file.seek(0, 0) child_exit_code = None # The JSON status file's output is a JSON object per line diff --git a/src/buildstream/sandbox/_sandboxchroot.py b/src/buildstream/sandbox/_sandboxchroot.py index 7266a00e3..084ed5b6c 100644 --- a/src/buildstream/sandbox/_sandboxchroot.py +++ b/src/buildstream/sandbox/_sandboxchroot.py @@ -26,7 +26,7 @@ import subprocess from contextlib import contextmanager, ExitStack import psutil -from .._exceptions import SandboxError +from .._exceptions import SandboxError, PlatformError from .. import utils from .. import _signals from ._mounter import Mounter @@ -35,7 +35,6 @@ from . import Sandbox, SandboxFlags class SandboxChroot(Sandbox): - _FUSE_MOUNT_OPTIONS = {'dev': True} def __init__(self, *args, **kwargs): @@ -49,6 +48,33 @@ class SandboxChroot(Sandbox): self.mount_map = None + @classmethod + def check_available(cls): + cls._uid = os.getuid() + cls._gid = os.getgid() + + available = cls._uid == 0 + if not available: + cls._dummy_reasons += ["uid is not 0"] + raise SandboxError("can not run chroot if uid is not 0") + + @classmethod + def check_sandbox_config(cls, local_platform, config): + # With the chroot sandbox, the UID/GID in the sandbox + # will match the host UID/GID (typically 0/0). + if config.build_uid != cls._uid or config.build_gid != cls._gid: + return False + + host_os = local_platform.get_host_os() + host_arch = local_platform.get_host_arch() + # Check host os and architecture match + if config.build_os != host_os: + raise PlatformError("Configured and host OS don't match.") + elif config.build_arch != host_arch: + raise PlatformError("Configured and host architecture don't match.") + + return True + def _run(self, command, flags, *, cwd, env): if not self._has_command(command[0], env): diff --git a/src/buildstream/sandbox/sandbox.py b/src/buildstream/sandbox/sandbox.py index a651fb783..4cab7d6b8 100644 --- a/src/buildstream/sandbox/sandbox.py +++ b/src/buildstream/sandbox/sandbox.py @@ -108,6 +108,7 @@ class Sandbox(): '/dev/zero', '/dev/null' ] + _dummy_reasons = [] def __init__(self, context, project, directory, **kwargs): self.__context = context diff --git a/src/buildstream/testing/_utils/site.py b/src/buildstream/testing/_utils/site.py index 64e0603ab..d51d37525 100644 --- a/src/buildstream/testing/_utils/site.py +++ b/src/buildstream/testing/_utils/site.py @@ -63,17 +63,21 @@ try: except ImportError: HAVE_ARPY = False +try: + utils.get_host_tool('buildbox') + HAVE_BUILDBOX = True +except ProgramNotFoundError: + HAVE_BUILDBOX = False + IS_LINUX = os.getenv('BST_FORCE_BACKEND', sys.platform).startswith('linux') IS_WSL = (IS_LINUX and 'Microsoft' in platform.uname().release) IS_WINDOWS = (os.name == 'nt') -if not IS_LINUX: - HAVE_SANDBOX = True # fallback to a chroot sandbox on unix -elif IS_WSL: - HAVE_SANDBOX = False # Sandboxes are inoperable under WSL due to lack of FUSE -elif IS_LINUX and HAVE_BWRAP: - HAVE_SANDBOX = True -else: - HAVE_SANDBOX = False - MACHINE_ARCH = Platform.get_host_arch() + +HAVE_SANDBOX = os.getenv('BST_FORCE_SANDBOX') + +if HAVE_SANDBOX is not None: + pass +elif IS_LINUX and HAVE_BWRAP and (not IS_WSL): + HAVE_SANDBOX = 'bwrap' diff --git a/src/buildstream/testing/runcli.py b/src/buildstream/testing/runcli.py index 8e9065478..02334aa53 100644 --- a/src/buildstream/testing/runcli.py +++ b/src/buildstream/testing/runcli.py @@ -775,10 +775,7 @@ def cli_integration(tmpdir, integration_cache): directory = os.path.join(str(tmpdir), 'cache') os.makedirs(directory) - if os.environ.get('BST_FORCE_BACKEND') == 'unix': - fixture = CliIntegration(directory, default_options=[('linux', 'False')]) - else: - fixture = CliIntegration(directory) + fixture = CliIntegration(directory) # We want to cache sources for integration tests more permanently, # to avoid downloading the huge base-sdk repeatedly diff --git a/tests/integration/build-uid.py b/tests/integration/build-uid.py index 26f1bd2d4..66f9b3fbc 100644 --- a/tests/integration/build-uid.py +++ b/tests/integration/build-uid.py @@ -5,7 +5,7 @@ import os import pytest from buildstream.testing import cli_integration as cli # pylint: disable=unused-import -from buildstream.testing._utils.site import HAVE_BWRAP, HAVE_SANDBOX, IS_LINUX +from buildstream.testing._utils.site import HAVE_SANDBOX, IS_LINUX pytestmark = pytest.mark.integration @@ -16,7 +16,7 @@ DATA_DIR = os.path.join( ) -@pytest.mark.skipif(not IS_LINUX or not HAVE_BWRAP, reason='Only available on linux with bubblewrap') +@pytest.mark.skipif(not IS_LINUX or HAVE_SANDBOX != "bwrap", reason='Only available on linux with bubblewrap') @pytest.mark.datafiles(DATA_DIR) def test_build_uid_overridden(cli, datafiles): project = str(datafiles) @@ -35,7 +35,7 @@ def test_build_uid_overridden(cli, datafiles): assert result.exit_code == 0 -@pytest.mark.skipif(not IS_LINUX or not HAVE_BWRAP, reason='Only available on linux with bubbelwrap') +@pytest.mark.skipif(not IS_LINUX or HAVE_SANDBOX != "bwrap", reason='Only available on linux with bubbelwrap') @pytest.mark.datafiles(DATA_DIR) def test_build_uid_in_project(cli, datafiles): project = str(datafiles) @@ -55,7 +55,7 @@ def test_build_uid_in_project(cli, datafiles): @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(HAVE_SANDBOX != "bwrap", reason='Only available with a functioning sandbox') def test_build_uid_default(cli, datafiles): project = str(datafiles) element_name = 'build-uid/build-uid-default.bst' diff --git a/tests/integration/cachedfail.py b/tests/integration/cachedfail.py index a2273a06d..be7db3357 100644 --- a/tests/integration/cachedfail.py +++ b/tests/integration/cachedfail.py @@ -1,3 +1,19 @@ +# +# Copyright (C) 2016 Codethink Limited +# Copyright (C) 2019 Bloomberg Finance LP +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name @@ -7,7 +23,7 @@ import pytest from buildstream import _yaml from buildstream._exceptions import ErrorDomain from buildstream.testing import cli_integration as cli # pylint: disable=unused-import -from buildstream.testing._utils.site import HAVE_BWRAP, HAVE_SANDBOX, IS_LINUX +from buildstream.testing._utils.site import HAVE_SANDBOX from tests.conftest import clean_platform_cache from tests.testutils import create_artifact_share @@ -167,7 +183,7 @@ def test_push_cached_fail(cli, tmpdir, datafiles, on_error): assert share.has_artifact(cli.get_artifact_name(project, 'test', 'element.bst')) -@pytest.mark.skipif(not (IS_LINUX and HAVE_BWRAP), reason='Only available with bubblewrap on Linux') +@pytest.mark.skipif(HAVE_SANDBOX != 'bwrap', reason='Only available with bubblewrap on Linux') @pytest.mark.datafiles(DATA_DIR) def test_host_tools_errors_are_not_cached(cli, datafiles): project = str(datafiles) @@ -190,8 +206,10 @@ def test_host_tools_errors_are_not_cached(cli, datafiles): } _yaml.dump(element, element_path) + clean_platform_cache() + # Build without access to host tools, this will fail - result1 = cli.run(project=project, args=['build', 'element.bst'], env={'PATH': ''}) + result1 = cli.run(project=project, args=['build', 'element.bst'], env={'PATH': '', 'BST_FORCE_SANDBOX': None}) result1.assert_task_error(ErrorDomain.SANDBOX, 'unavailable-local-sandbox') assert cli.get_element_state(project, 'element.bst') == 'buildable' diff --git a/tests/integration/sandbox-bwrap.py b/tests/integration/sandbox-bwrap.py index d08076f5a..f48c75cbd 100644 --- a/tests/integration/sandbox-bwrap.py +++ b/tests/integration/sandbox-bwrap.py @@ -7,7 +7,7 @@ import pytest from buildstream._exceptions import ErrorDomain from buildstream.testing import cli_integration as cli # pylint: disable=unused-import -from buildstream.testing._utils.site import HAVE_BWRAP, HAVE_BWRAP_JSON_STATUS +from buildstream.testing._utils.site import HAVE_SANDBOX, HAVE_BWRAP_JSON_STATUS pytestmark = pytest.mark.integration @@ -22,7 +22,7 @@ DATA_DIR = os.path.join( # Bubblewrap sandbox doesn't remove the dirs it created during its execution, # so BuildStream tries to remove them to do good. BuildStream should be extra # careful when those folders already exist and should not touch them, though. -@pytest.mark.skipif(not HAVE_BWRAP, reason='Only available with bubblewrap') +@pytest.mark.skipif(HAVE_SANDBOX != 'bwrap', reason='Only available with bubblewrap') @pytest.mark.datafiles(DATA_DIR) def test_sandbox_bwrap_cleanup_build(cli, datafiles): project = str(datafiles) @@ -34,7 +34,7 @@ def test_sandbox_bwrap_cleanup_build(cli, datafiles): assert result.exit_code == 0 -@pytest.mark.skipif(not HAVE_BWRAP, reason='Only available with bubblewrap') +@pytest.mark.skipif(HAVE_SANDBOX != 'bwrap', reason='Only available with bubblewrap') @pytest.mark.skipif(not HAVE_BWRAP_JSON_STATUS, reason='Only available with bubblewrap supporting --json-status-fd') @pytest.mark.datafiles(DATA_DIR) def test_sandbox_bwrap_distinguish_setup_error(cli, datafiles): @@ -45,7 +45,7 @@ def test_sandbox_bwrap_distinguish_setup_error(cli, datafiles): result.assert_task_error(error_domain=ErrorDomain.SANDBOX, error_reason="bwrap-sandbox-fail") -@pytest.mark.skipif(not HAVE_BWRAP, reason='Only available with bubblewrap') +@pytest.mark.skipif(HAVE_SANDBOX != 'bwrap', reason='Only available with bubblewrap') @pytest.mark.datafiles(DATA_DIR) def test_sandbox_bwrap_return_subprocess(cli, datafiles): project = str(datafiles) diff --git a/tests/sandboxes/missing_dependencies.py b/tests/sandboxes/missing_dependencies.py index ee346010e..79153f769 100644 --- a/tests/sandboxes/missing_dependencies.py +++ b/tests/sandboxes/missing_dependencies.py @@ -43,7 +43,10 @@ def test_missing_brwap_has_nice_error_message(cli, datafiles): # Build without access to host tools, this should fail with a nice error result = cli.run( - project=project, args=['build', 'element.bst'], env={'PATH': ''}) + project=project, + args=['build', 'element.bst'], + env={'PATH': '', 'BST_FORCE_SANDBOX': None} + ) result.assert_task_error(ErrorDomain.SANDBOX, 'unavailable-local-sandbox') assert "not found" in result.stderr @@ -85,6 +88,7 @@ def test_old_brwap_has_nice_error_message(cli, datafiles, tmp_path): result = cli.run( project=project, args=['--debug', '--verbose', 'build', 'element3.bst'], - env={'PATH': str(tmp_path.joinpath('bin'))}) + env={'PATH': str(tmp_path.joinpath('bin')), + 'BST_FORCE_SANDBOX': None}) result.assert_task_error(ErrorDomain.SANDBOX, 'unavailable-local-sandbox') assert "too old" in result.stderr @@ -36,6 +36,7 @@ deps = passenv = ARTIFACT_CACHE_SERVICE BST_FORCE_BACKEND + BST_FORCE_SANDBOX GI_TYPELIB_PATH INTEGRATION_CACHE http_proxy |