Move source from 'buildstream' to 'src/buildstream'

This was discussed in #1008.

Fixes #1009.

5 files changed, 438 insertions, 0 deletions

diff --git a/src/buildstream/_platform/__init__.py b/src/buildstream/_platform/__init__.py
new file mode 100644
index 000000000..29a29894b
--- /dev/null
+++ b/src/buildstream/_platform/__init__.py
@@ -0,0 +1,20 @@
+#
+#  Copyright (C) 2017 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Maat <tristan.maat@codethink.co.uk>
+
+from .platform import Platform
diff --git a/src/buildstream/_platform/darwin.py b/src/buildstream/_platform/darwin.py
new file mode 100644
index 000000000..8e08685ec
--- /dev/null
+++ b/src/buildstream/_platform/darwin.py
@@ -0,0 +1,48 @@
+#
+#  Copyright (C) 2017 Codethink Limited
+#  Copyright (C) 2018 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+
+from ..sandbox import SandboxDummy
+
+from .platform import Platform
+
+
+class Darwin(Platform):
+
+    # This value comes from OPEN_MAX in syslimits.h
+    OPEN_MAX = 10240
+
+    def create_sandbox(self, *args, **kwargs):
+        kwargs['dummy_reason'] = \
+            "OSXFUSE is not supported and there are no supported sandbox " + \
+            "technologies for MacOS at this time"
+        return SandboxDummy(*args, **kwargs)
+
+    def check_sandbox_config(self, config):
+        # Accept all sandbox configs as it's irrelevant with the dummy sandbox (no Sandbox.run).
+        return True
+
+    def get_cpu_count(self, cap=None):
+        cpu_count = os.cpu_count()
+        if cap is None:
+            return cpu_count
+        else:
+            return min(cpu_count, cap)
+
+    def set_resource_limits(self, soft_limit=OPEN_MAX, hard_limit=None):
+        super().set_resource_limits(soft_limit)
diff --git a/src/buildstream/_platform/linux.py b/src/buildstream/_platform/linux.py
new file mode 100644
index 000000000..e4ce02572
--- /dev/null
+++ b/src/buildstream/_platform/linux.py
@@ -0,0 +1,150 @@
+#
+#  Copyright (C) 2017 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Maat <tristan.maat@codethink.co.uk>
+
+import os
+import subprocess
+
+from .. import _site
+from .. import utils
+from ..sandbox import SandboxDummy
+
+from .platform import Platform
+from .._exceptions import PlatformError
+
+
+class Linux(Platform):
+
+    def __init__(self):
+
+        super().__init__()
+
+        self._uid = os.geteuid()
+        self._gid = os.getegid()
+
+        self._have_fuse = os.path.exists("/dev/fuse")
+
+        bwrap_version = _site.get_bwrap_version()
+
+        if bwrap_version is None:
+            self._bwrap_exists = False
+            self._have_good_bwrap = False
+            self._die_with_parent_available = False
+            self._json_status_available = False
+        else:
+            self._bwrap_exists = True
+            self._have_good_bwrap = (0, 1, 2) <= bwrap_version
+            self._die_with_parent_available = (0, 1, 8) <= bwrap_version
+            self._json_status_available = (0, 3, 2) <= bwrap_version
+
+        self._local_sandbox_available = self._have_fuse and self._have_good_bwrap
+
+        if self._local_sandbox_available:
+            self._user_ns_available = self._check_user_ns_available()
+        else:
+            self._user_ns_available = False
+
+        # Set linux32 option
+        self._linux32 = False
+
+    def create_sandbox(self, *args, **kwargs):
+        if not self._local_sandbox_available:
+            return self._create_dummy_sandbox(*args, **kwargs)
+        else:
+            return self._create_bwrap_sandbox(*args, **kwargs)
+
+    def check_sandbox_config(self, config):
+        if not self._local_sandbox_available:
+            # Accept all sandbox configs as it's irrelevant with the dummy sandbox (no Sandbox.run).
+            return True
+
+        if self._user_ns_available:
+            # User namespace support allows arbitrary build UID/GID settings.
+            pass
+        elif (config.build_uid != self._uid or config.build_gid != self._gid):
+            # Without user namespace support, the UID/GID in the sandbox
+            # will match the host UID/GID.
+            return False
+
+        # We can't do builds for another host or architecture except x86-32 on
+        # x86-64
+        host_os = self.get_host_os()
+        host_arch = self.get_host_arch()
+        if config.build_os != host_os:
+            raise PlatformError("Configured and host OS don't match.")
+        elif config.build_arch != host_arch:
+            # We can use linux32 for building 32bit on 64bit machines
+            if (host_os == "Linux" and
+                    ((config.build_arch == "x86-32" and host_arch == "x86-64") or
+                     (config.build_arch == "aarch32" and host_arch == "aarch64"))):
+                # check linux32 is available
+                try:
+                    utils.get_host_tool('linux32')
+                    self._linux32 = True
+                except utils.ProgramNotFoundError:
+                    pass
+            else:
+                raise PlatformError("Configured architecture and host architecture don't match.")
+
+        return True
+
+    ################################################
+    #              Private Methods                 #
+    ################################################
+
+    def _create_dummy_sandbox(self, *args, **kwargs):
+        reasons = []
+        if not self._have_fuse:
+            reasons.append("FUSE is unavailable")
+        if not self._have_good_bwrap:
+            if self._bwrap_exists:
+                reasons.append("`bwrap` is too old (bst needs at least 0.1.2)")
+            else:
+                reasons.append("`bwrap` executable not found")
+
+        kwargs['dummy_reason'] = " and ".join(reasons)
+        return SandboxDummy(*args, **kwargs)
+
+    def _create_bwrap_sandbox(self, *args, **kwargs):
+        from ..sandbox._sandboxbwrap import SandboxBwrap
+        # Inform the bubblewrap sandbox as to whether it can use user namespaces or not
+        kwargs['user_ns_available'] = self._user_ns_available
+        kwargs['die_with_parent_available'] = self._die_with_parent_available
+        kwargs['json_status_available'] = self._json_status_available
+        kwargs['linux32'] = self._linux32
+        return SandboxBwrap(*args, **kwargs)
+
+    def _check_user_ns_available(self):
+        # Here, lets check if bwrap is able to create user namespaces,
+        # issue a warning if it's not available, and save the state
+        # locally so that we can inform the sandbox to not try it
+        # later on.
+        bwrap = utils.get_host_tool('bwrap')
+        whoami = utils.get_host_tool('whoami')
+        try:
+            output = subprocess.check_output([
+                bwrap,
+                '--ro-bind', '/', '/',
+                '--unshare-user',
+                '--uid', '0', '--gid', '0',
+                whoami,
+            ], universal_newlines=True).strip()
+        except subprocess.CalledProcessError:
+            output = ''
+
+        return output == 'root'
diff --git a/src/buildstream/_platform/platform.py b/src/buildstream/_platform/platform.py
new file mode 100644
index 000000000..dba60ddca
--- /dev/null
+++ b/src/buildstream/_platform/platform.py
@@ -0,0 +1,164 @@
+#
+#  Copyright (C) 2017 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Maat <tristan.maat@codethink.co.uk>
+
+import os
+import platform
+import sys
+import resource
+
+from .._exceptions import PlatformError, ImplError
+
+
+class Platform():
+    _instance = None
+
+    # Platform()
+    #
+    # A class to manage platform-specific details. Currently holds the
+    # sandbox factory as well as platform helpers.
+    #
+    def __init__(self):
+        self.set_resource_limits()
+
+    @classmethod
+    def _create_instance(cls):
+        # Meant for testing purposes and therefore hidden in the
+        # deepest corners of the source code. Try not to abuse this,
+        # please?
+        if os.getenv('BST_FORCE_BACKEND'):
+            backend = os.getenv('BST_FORCE_BACKEND')
+        elif sys.platform.startswith('linux'):
+            backend = 'linux'
+        elif sys.platform.startswith('darwin'):
+            backend = 'darwin'
+        else:
+            backend = 'unix'
+
+        if backend == 'linux':
+            from .linux import Linux as PlatformImpl  # pylint: disable=cyclic-import
+        elif backend == 'darwin':
+            from .darwin import Darwin as PlatformImpl  # pylint: disable=cyclic-import
+        elif backend == 'unix':
+            from .unix import Unix as PlatformImpl  # pylint: disable=cyclic-import
+        else:
+            raise PlatformError("No such platform: '{}'".format(backend))
+
+        cls._instance = PlatformImpl()
+
+    @classmethod
+    def get_platform(cls):
+        if not cls._instance:
+            cls._create_instance()
+        return cls._instance
+
+    def get_cpu_count(self, cap=None):
+        cpu_count = len(os.sched_getaffinity(0))
+        if cap is None:
+            return cpu_count
+        else:
+            return min(cpu_count, cap)
+
+    @staticmethod
+    def get_host_os():
+        return platform.uname().system
+
+    # canonicalize_arch():
+    #
+    # This returns the canonical, OS-independent architecture name
+    # or raises a PlatformError if the architecture is unknown.
+    #
+    @staticmethod
+    def canonicalize_arch(arch):
+        # Note that these are all expected to be lowercase, as we want a
+        # case-insensitive lookup. Windows can report its arch in ALLCAPS.
+        aliases = {
+            "aarch32": "aarch32",
+            "aarch64": "aarch64",
+            "aarch64-be": "aarch64-be",
+            "amd64": "x86-64",
+            "arm": "aarch32",
+            "armv8l": "aarch64",
+            "armv8b": "aarch64-be",
+            "i386": "x86-32",
+            "i486": "x86-32",
+            "i586": "x86-32",
+            "i686": "x86-32",
+            "power-isa-be": "power-isa-be",
+            "power-isa-le": "power-isa-le",
+            "ppc64": "power-isa-be",
+            "ppc64le": "power-isa-le",
+            "sparc": "sparc-v9",
+            "sparc64": "sparc-v9",
+            "sparc-v9": "sparc-v9",
+            "x86-32": "x86-32",
+            "x86-64": "x86-64"
+        }
+
+        try:
+            return aliases[arch.replace('_', '-').lower()]
+        except KeyError:
+            raise PlatformError("Unknown architecture: {}".format(arch))
+
+    # get_host_arch():
+    #
+    # This returns the architecture of the host machine. The possible values
+    # map from uname -m in order to be a OS independent list.
+    #
+    # Returns:
+    #    (string): String representing the architecture
+    @staticmethod
+    def get_host_arch():
+        # get the hardware identifier from uname
+        uname_machine = platform.uname().machine
+        return Platform.canonicalize_arch(uname_machine)
+
+    ##################################################################
+    #                        Sandbox functions                       #
+    ##################################################################
+
+    # create_sandbox():
+    #
+    # Create a build sandbox suitable for the environment
+    #
+    # Args:
+    #     args (dict): The arguments to pass to the sandbox constructor
+    #     kwargs (file): The keyword arguments to pass to the sandbox constructor
+    #
+    # Returns:
+    #     (Sandbox) A sandbox
+    #
+    def create_sandbox(self, *args, **kwargs):
+        raise ImplError("Platform {platform} does not implement create_sandbox()"
+                        .format(platform=type(self).__name__))
+
+    def check_sandbox_config(self, config):
+        raise ImplError("Platform {platform} does not implement check_sandbox_config()"
+                        .format(platform=type(self).__name__))
+
+    def set_resource_limits(self, soft_limit=None, hard_limit=None):
+        # Need to set resources for _frontend/app.py as this is dependent on the platform
+        # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
+        # Avoid hitting the limit too quickly.
+        limits = resource.getrlimit(resource.RLIMIT_NOFILE)
+        if limits[0] != limits[1]:
+            if soft_limit is None:
+                soft_limit = limits[1]
+            if hard_limit is None:
+                hard_limit = limits[1]
+            resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit))
diff --git a/src/buildstream/_platform/unix.py b/src/buildstream/_platform/unix.py
new file mode 100644
index 000000000..d04b0712c
--- /dev/null
+++ b/src/buildstream/_platform/unix.py
@@ -0,0 +1,56 @@
+#
+#  Copyright (C) 2017 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Maat <tristan.maat@codethink.co.uk>
+
+import os
+
+from .._exceptions import PlatformError
+
+from .platform import Platform
+
+
+class Unix(Platform):
+
+    def __init__(self):
+
+        super().__init__()
+
+        self._uid = os.geteuid()
+        self._gid = os.getegid()
+
+        # Not necessarily 100% reliable, but we want to fail early.
+        if self._uid != 0:
+            raise PlatformError("Root privileges are required to run without bubblewrap.")
+
+    def create_sandbox(self, *args, **kwargs):
+        from ..sandbox._sandboxchroot import SandboxChroot
+        return SandboxChroot(*args, **kwargs)
+
+    def check_sandbox_config(self, config):
+        # With the chroot sandbox, the UID/GID in the sandbox
+        # will match the host UID/GID (typically 0/0).
+        if config.build_uid != self._uid or config.build_gid != self._gid:
+            return False
+
+        # Check host os and architecture match
+        if config.build_os != self.get_host_os():
+            raise PlatformError("Configured and host OS don't match.")
+        elif config.build_arch != self.get_host_arch():
+            raise PlatformError("Configured and host architecture don't match.")
+
+        return True