Refactor of Platform and Sandbox

5 files changed, 184 insertions, 176 deletions

diff --git a/src/buildstream/_platform/darwin.py b/src/buildstream/_platform/darwin.py
index 282a5b445..e8c1ffaf3 100644
--- a/src/buildstream/_platform/darwin.py
+++ b/src/buildstream/_platform/darwin.py
@@ -28,16 +28,6 @@ class Darwin(Platform):
     # This value comes from OPEN_MAX in syslimits.h
     OPEN_MAX = 10240
 
-    def create_sandbox(self, *args, **kwargs):
-        kwargs['dummy_reason'] = \
-            "OSXFUSE is not supported and there are no supported sandbox " + \
-            "technologies for MacOS at this time"
-        return SandboxDummy(*args, **kwargs)
-
-    def check_sandbox_config(self, config):
-        # Accept all sandbox configs as it's irrelevant with the dummy sandbox (no Sandbox.run).
-        return True
-
     def get_cpu_count(self, cap=None):
         cpu_count = os.cpu_count()
         if cap is None:
@@ -62,3 +52,17 @@ class Darwin(Platform):
         old_soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
         soft_limit = min(max(self.OPEN_MAX, old_soft_limit), hard_limit)
         resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit))
+
+    def _setup_dummy_sandbox(self):
+        def _check_dummy_sandbox_config(config):
+            return True
+        self.check_sandbox_config = _check_dummy_sandbox_config
+
+        def _create_dummy_sandbox(*args, **kwargs):
+            kwargs['dummy_reason'] = \
+                "OSXFUSE is not supported and there are no supported sandbox " + \
+                "technologies for MacOS at this time"
+            return SandboxDummy(*args, **kwargs)
+        self.create_sandbox = _create_dummy_sandbox
+
+        return True
diff --git a/src/buildstream/_platform/fallback.py b/src/buildstream/_platform/fallback.py
new file mode 100644
index 000000000..39669e0c2
--- /dev/null
+++ b/src/buildstream/_platform/fallback.py
@@ -0,0 +1,37 @@
+#
+#  Copyright (C) 2018 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+
+from ..sandbox import SandboxDummy
+
+from .platform import Platform
+
+
+class Fallback(Platform):
+
+    def _setup_dummy_sandbox(self):
+        def _check_dummy_sandbox_config(config):
+            return True
+        self.check_sandbox_config = _check_dummy_sandbox_config
+
+        def _create_dummy_sandbox(*args, **kwargs):
+            kwargs['dummy_reason'] = \
+                ("FallBack platform only implements dummy sandbox, "
+                 "Buildstream may be having issues correctly detecting your platform, platform "
+                 "can be forced with BST_FORCE_BACKEND")
+            return SandboxDummy(*args, **kwargs)
+        self.create_sandbox = _create_dummy_sandbox
+
+        return True
diff --git a/src/buildstream/_platform/linux.py b/src/buildstream/_platform/linux.py
index e4ce02572..3d85fdf34 100644
--- a/src/buildstream/_platform/linux.py
+++ b/src/buildstream/_platform/linux.py
@@ -1,5 +1,6 @@
 #
 #  Copyright (C) 2017 Codethink Limited
+#  Copyright (C) 2018 Bloomberg Finance LP
 #
 #  This program is free software; you can redistribute it and/or
 #  modify it under the terms of the GNU Lesser General Public
@@ -18,133 +19,100 @@
 #        Tristan Maat <tristan.maat@codethink.co.uk>
 
 import os
-import subprocess
 
-from .. import _site
 from .. import utils
 from ..sandbox import SandboxDummy
 
 from .platform import Platform
-from .._exceptions import PlatformError
 
 
 class Linux(Platform):
 
-    def __init__(self):
+    def _setup_sandbox(self, force_sandbox):
+        sandbox_setups = {
+            'bwrap': self._setup_bwrap_sandbox,
+            'chroot': self._setup_chroot_sandbox,
+            'dummy': self._setup_dummy_sandbox,
+        }
 
-        super().__init__()
+        preferred_sandboxes = [
+            'bwrap',
+        ]
 
-        self._uid = os.geteuid()
-        self._gid = os.getegid()
-
-        self._have_fuse = os.path.exists("/dev/fuse")
-
-        bwrap_version = _site.get_bwrap_version()
+        self._try_sandboxes(force_sandbox, sandbox_setups, preferred_sandboxes)
 
-        if bwrap_version is None:
-            self._bwrap_exists = False
-            self._have_good_bwrap = False
-            self._die_with_parent_available = False
-            self._json_status_available = False
-        else:
-            self._bwrap_exists = True
-            self._have_good_bwrap = (0, 1, 2) <= bwrap_version
-            self._die_with_parent_available = (0, 1, 8) <= bwrap_version
-            self._json_status_available = (0, 3, 2) <= bwrap_version
+    def __init__(self, force_sandbox=None):
+        super().__init__(force_sandbox=force_sandbox)
 
-        self._local_sandbox_available = self._have_fuse and self._have_good_bwrap
-
-        if self._local_sandbox_available:
-            self._user_ns_available = self._check_user_ns_available()
-        else:
-            self._user_ns_available = False
+        self._uid = os.geteuid()
+        self._gid = os.getegid()
 
         # Set linux32 option
-        self._linux32 = False
+        self.linux32 = None
 
-    def create_sandbox(self, *args, **kwargs):
-        if not self._local_sandbox_available:
-            return self._create_dummy_sandbox(*args, **kwargs)
-        else:
-            return self._create_bwrap_sandbox(*args, **kwargs)
-
-    def check_sandbox_config(self, config):
-        if not self._local_sandbox_available:
-            # Accept all sandbox configs as it's irrelevant with the dummy sandbox (no Sandbox.run).
-            return True
-
-        if self._user_ns_available:
-            # User namespace support allows arbitrary build UID/GID settings.
-            pass
-        elif (config.build_uid != self._uid or config.build_gid != self._gid):
-            # Without user namespace support, the UID/GID in the sandbox
-            # will match the host UID/GID.
-            return False
-
-        # We can't do builds for another host or architecture except x86-32 on
-        # x86-64
-        host_os = self.get_host_os()
+    def can_crossbuild(self, config):
         host_arch = self.get_host_arch()
-        if config.build_os != host_os:
-            raise PlatformError("Configured and host OS don't match.")
-        elif config.build_arch != host_arch:
-            # We can use linux32 for building 32bit on 64bit machines
-            if (host_os == "Linux" and
-                    ((config.build_arch == "x86-32" and host_arch == "x86-64") or
-                     (config.build_arch == "aarch32" and host_arch == "aarch64"))):
-                # check linux32 is available
+        if ((config.build_arch == "x86-32" and host_arch == "x86-64") or
+                (config.build_arch == "aarch32" and host_arch == "aarch64")):
+            if self.linux32 is None:
                 try:
                     utils.get_host_tool('linux32')
-                    self._linux32 = True
+                    self.linux32 = True
                 except utils.ProgramNotFoundError:
-                    pass
-            else:
-                raise PlatformError("Configured architecture and host architecture don't match.")
-
-        return True
+                    self.linux32 = False
+            return self.linux32
+        return False
 
     ################################################
     #              Private Methods                 #
     ################################################
 
-    def _create_dummy_sandbox(self, *args, **kwargs):
-        reasons = []
-        if not self._have_fuse:
-            reasons.append("FUSE is unavailable")
-        if not self._have_good_bwrap:
-            if self._bwrap_exists:
-                reasons.append("`bwrap` is too old (bst needs at least 0.1.2)")
-            else:
-                reasons.append("`bwrap` executable not found")
+    def _setup_dummy_sandbox(self):
+        dummy_reasons = " and ".join(self.dummy_reasons)
+
+        def _check_dummy_sandbox_config(config):
+            return True
+        self.check_sandbox_config = _check_dummy_sandbox_config
+
+        def _create_dummy_sandbox(*args, **kwargs):
+            kwargs['dummy_reason'] = dummy_reasons
+            return SandboxDummy(*args, **kwargs)
+        self.create_sandbox = _create_dummy_sandbox
 
-        kwargs['dummy_reason'] = " and ".join(reasons)
-        return SandboxDummy(*args, **kwargs)
+        return True
 
-    def _create_bwrap_sandbox(self, *args, **kwargs):
+    def _setup_bwrap_sandbox(self):
         from ..sandbox._sandboxbwrap import SandboxBwrap
-        # Inform the bubblewrap sandbox as to whether it can use user namespaces or not
-        kwargs['user_ns_available'] = self._user_ns_available
-        kwargs['die_with_parent_available'] = self._die_with_parent_available
-        kwargs['json_status_available'] = self._json_status_available
-        kwargs['linux32'] = self._linux32
-        return SandboxBwrap(*args, **kwargs)
-
-    def _check_user_ns_available(self):
-        # Here, lets check if bwrap is able to create user namespaces,
-        # issue a warning if it's not available, and save the state
-        # locally so that we can inform the sandbox to not try it
-        # later on.
-        bwrap = utils.get_host_tool('bwrap')
-        whoami = utils.get_host_tool('whoami')
-        try:
-            output = subprocess.check_output([
-                bwrap,
-                '--ro-bind', '/', '/',
-                '--unshare-user',
-                '--uid', '0', '--gid', '0',
-                whoami,
-            ], universal_newlines=True).strip()
-        except subprocess.CalledProcessError:
-            output = ''
-
-        return output == 'root'
+
+        # This function should only be called once.
+        # but if it does eg, in the tests we want to
+        # reset the sandbox checks
+
+        SandboxBwrap._have_good_bwrap = None
+        self._check_sandbox(SandboxBwrap)
+
+        def _check_sandbox_config_bwrap(config):
+            return SandboxBwrap.check_sandbox_config(self, config)
+        self.check_sandbox_config = _check_sandbox_config_bwrap
+
+        def _create_bwrap_sandbox(*args, **kwargs):
+            kwargs['linux32'] = self.linux32
+            return SandboxBwrap(*args, **kwargs)
+        self.create_sandbox = _create_bwrap_sandbox
+
+        return True
+
+    def _setup_chroot_sandbox(self):
+        from ..sandbox._sandboxchroot import SandboxChroot
+
+        self._check_sandbox(SandboxChroot)
+
+        def _check_sandbox_config_chroot(config):
+            return SandboxChroot.check_sandbox_config(self, config)
+        self.check_sandbox_config = _check_sandbox_config_chroot
+
+        def _create_chroot_sandbox(*args, **kwargs):
+            return SandboxChroot(*args, **kwargs)
+        self.create_sandbox = _create_chroot_sandbox
+
+        return True
diff --git a/src/buildstream/_platform/platform.py b/src/buildstream/_platform/platform.py
index 5f2b7081a..dab6049ea 100644
--- a/src/buildstream/_platform/platform.py
+++ b/src/buildstream/_platform/platform.py
@@ -1,5 +1,6 @@
 #
 #  Copyright (C) 2017 Codethink Limited
+#  Copyright (C) 2018 Bloomberg Finance LP
 #
 #  This program is free software; you can redistribute it and/or
 #  modify it under the terms of the GNU Lesser General Public
@@ -23,7 +24,8 @@ import sys
 
 import psutil
 
-from .._exceptions import PlatformError, ImplError
+from .._exceptions import PlatformError, ImplError, SandboxError
+from .. import utils
 
 
 class Platform():
@@ -34,33 +36,82 @@ class Platform():
     # A class to manage platform-specific details. Currently holds the
     # sandbox factory as well as platform helpers.
     #
-    def __init__(self):
+    # Args:
+    #     force_sandbox (bool): Force bst to use a particular sandbox
+    #
+    def __init__(self, force_sandbox=None):
         self.maximize_open_file_limit()
+        self._local_sandbox = None
+        self.dummy_reasons = []
+        self._setup_sandbox(force_sandbox)
+
+    def _setup_sandbox(self, force_sandbox):
+        sandbox_setups = {'dummy': self._setup_dummy_sandbox}
+        preferred_sandboxes = []
+        self._try_sandboxes(force_sandbox, sandbox_setups, preferred_sandboxes)
+
+    def _try_sandboxes(self, force_sandbox, sandbox_setups, preferred_sandboxes):
+        # Any sandbox from sandbox_setups can be forced by BST_FORCE_SANDBOX
+        # But if a specific sandbox is not forced then only `first class` sandbox are tried before
+        # falling back to the dummy sandbox.
+        # Where `first_class` sandboxes are those in preferred_sandboxes
+        if force_sandbox:
+            try:
+                sandbox_setups[force_sandbox]()
+            except KeyError:
+                raise PlatformError("Forced Sandbox is unavailable on this platform: BST_FORCE_SANDBOX"
+                                    " is set to {} but it is not available".format(force_sandbox))
+            except SandboxError as Error:
+                raise PlatformError("Forced Sandbox Error: BST_FORCE_SANDBOX"
+                                    " is set to {} but cannot be setup".format(force_sandbox),
+                                    detail=" and ".join(self.dummy_reasons)) from Error
+        else:
+            for good_sandbox in preferred_sandboxes:
+                try:
+                    sandbox_setups[good_sandbox]()
+                    return
+                except SandboxError:
+                    continue
+                except utils.ProgramNotFoundError:
+                    continue
+            sandbox_setups['dummy']()
+
+    def _check_sandbox(self, Sandbox):
+        try:
+            Sandbox.check_available()
+        except SandboxError as Error:
+            self.dummy_reasons += Sandbox._dummy_reasons
+            raise Error
 
     @classmethod
     def _create_instance(cls):
         # Meant for testing purposes and therefore hidden in the
         # deepest corners of the source code. Try not to abuse this,
         # please?
+        if os.getenv('BST_FORCE_SANDBOX'):
+            force_sandbox = os.getenv('BST_FORCE_SANDBOX')
+        else:
+            force_sandbox = None
+
         if os.getenv('BST_FORCE_BACKEND'):
             backend = os.getenv('BST_FORCE_BACKEND')
-        elif sys.platform.startswith('linux'):
-            backend = 'linux'
         elif sys.platform.startswith('darwin'):
             backend = 'darwin'
+        elif sys.platform.startswith('linux'):
+            backend = 'linux'
         else:
-            backend = 'unix'
+            backend = 'fallback'
 
         if backend == 'linux':
             from .linux import Linux as PlatformImpl  # pylint: disable=cyclic-import
         elif backend == 'darwin':
             from .darwin import Darwin as PlatformImpl  # pylint: disable=cyclic-import
-        elif backend == 'unix':
-            from .unix import Unix as PlatformImpl  # pylint: disable=cyclic-import
+        elif backend == 'fallback':
+            from .fallback import Fallback as PlatformImpl  # pylint: disable=cyclic-import
         else:
             raise PlatformError("No such platform: '{}'".format(backend))
 
-        cls._instance = PlatformImpl()
+        cls._instance = PlatformImpl(force_sandbox=force_sandbox)
 
     @classmethod
     def get_platform(cls):
@@ -167,3 +218,7 @@ class Platform():
         soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
         if soft_limit != hard_limit:
             resource.setrlimit(resource.RLIMIT_NOFILE, (hard_limit, hard_limit))
+
+    def _setup_dummy_sandbox(self):
+        raise ImplError("Platform {platform} does not implement _setup_dummy_sandbox()"
+                        .format(platform=type(self).__name__))
diff --git a/src/buildstream/_platform/unix.py b/src/buildstream/_platform/unix.py
deleted file mode 100644
index d04b0712c..000000000
--- a/src/buildstream/_platform/unix.py
+++ /dev/null
@@ -1,56 +0,0 @@
-#
-#  Copyright (C) 2017 Codethink Limited
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU Lesser General Public
-#  License as published by the Free Software Foundation; either
-#  version 2 of the License, or (at your option) any later version.
-#
-#  This library is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
-#  Lesser General Public License for more details.
-#
-#  You should have received a copy of the GNU Lesser General Public
-#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
-#
-#  Authors:
-#        Tristan Maat <tristan.maat@codethink.co.uk>
-
-import os
-
-from .._exceptions import PlatformError
-
-from .platform import Platform
-
-
-class Unix(Platform):
-
-    def __init__(self):
-
-        super().__init__()
-
-        self._uid = os.geteuid()
-        self._gid = os.getegid()
-
-        # Not necessarily 100% reliable, but we want to fail early.
-        if self._uid != 0:
-            raise PlatformError("Root privileges are required to run without bubblewrap.")
-
-    def create_sandbox(self, *args, **kwargs):
-        from ..sandbox._sandboxchroot import SandboxChroot
-        return SandboxChroot(*args, **kwargs)
-
-    def check_sandbox_config(self, config):
-        # With the chroot sandbox, the UID/GID in the sandbox
-        # will match the host UID/GID (typically 0/0).
-        if config.build_uid != self._uid or config.build_gid != self._gid:
-            return False
-
-        # Check host os and architecture match
-        if config.build_os != self.get_host_os():
-            raise PlatformError("Configured and host OS don't match.")
-        elif config.build_arch != self.get_host_arch():
-            raise PlatformError("Configured and host architecture don't match.")
-
-        return True