From 070d053e5cc47e572e9f9e647315082bd7a15c63 Mon Sep 17 00:00:00 2001 From: Chandan Singh Date: Wed, 24 Apr 2019 22:53:19 +0100 Subject: Move source from 'buildstream' to 'src/buildstream' This was discussed in #1008. Fixes #1009. --- src/buildstream/_platform/__init__.py | 20 +++++ src/buildstream/_platform/darwin.py | 48 ++++++++++ src/buildstream/_platform/linux.py | 150 +++++++++++++++++++++++++++++++ src/buildstream/_platform/platform.py | 164 ++++++++++++++++++++++++++++++++++ src/buildstream/_platform/unix.py | 56 ++++++++++++ 5 files changed, 438 insertions(+) create mode 100644 src/buildstream/_platform/__init__.py create mode 100644 src/buildstream/_platform/darwin.py create mode 100644 src/buildstream/_platform/linux.py create mode 100644 src/buildstream/_platform/platform.py create mode 100644 src/buildstream/_platform/unix.py (limited to 'src/buildstream/_platform') diff --git a/src/buildstream/_platform/__init__.py b/src/buildstream/_platform/__init__.py new file mode 100644 index 000000000..29a29894b --- /dev/null +++ b/src/buildstream/_platform/__init__.py @@ -0,0 +1,20 @@ +# +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . +# +# Authors: +# Tristan Maat + +from .platform import Platform diff --git a/src/buildstream/_platform/darwin.py b/src/buildstream/_platform/darwin.py new file mode 100644 index 000000000..8e08685ec --- /dev/null +++ b/src/buildstream/_platform/darwin.py @@ -0,0 +1,48 @@ +# +# Copyright (C) 2017 Codethink Limited +# Copyright (C) 2018 Bloomberg Finance LP +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . + +import os + +from ..sandbox import SandboxDummy + +from .platform import Platform + + +class Darwin(Platform): + + # This value comes from OPEN_MAX in syslimits.h + OPEN_MAX = 10240 + + def create_sandbox(self, *args, **kwargs): + kwargs['dummy_reason'] = \ + "OSXFUSE is not supported and there are no supported sandbox " + \ + "technologies for MacOS at this time" + return SandboxDummy(*args, **kwargs) + + def check_sandbox_config(self, config): + # Accept all sandbox configs as it's irrelevant with the dummy sandbox (no Sandbox.run). + return True + + def get_cpu_count(self, cap=None): + cpu_count = os.cpu_count() + if cap is None: + return cpu_count + else: + return min(cpu_count, cap) + + def set_resource_limits(self, soft_limit=OPEN_MAX, hard_limit=None): + super().set_resource_limits(soft_limit) diff --git a/src/buildstream/_platform/linux.py b/src/buildstream/_platform/linux.py new file mode 100644 index 000000000..e4ce02572 --- /dev/null +++ b/src/buildstream/_platform/linux.py @@ -0,0 +1,150 @@ +# +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . +# +# Authors: +# Tristan Maat + +import os +import subprocess + +from .. import _site +from .. import utils +from ..sandbox import SandboxDummy + +from .platform import Platform +from .._exceptions import PlatformError + + +class Linux(Platform): + + def __init__(self): + + super().__init__() + + self._uid = os.geteuid() + self._gid = os.getegid() + + self._have_fuse = os.path.exists("/dev/fuse") + + bwrap_version = _site.get_bwrap_version() + + if bwrap_version is None: + self._bwrap_exists = False + self._have_good_bwrap = False + self._die_with_parent_available = False + self._json_status_available = False + else: + self._bwrap_exists = True + self._have_good_bwrap = (0, 1, 2) <= bwrap_version + self._die_with_parent_available = (0, 1, 8) <= bwrap_version + self._json_status_available = (0, 3, 2) <= bwrap_version + + self._local_sandbox_available = self._have_fuse and self._have_good_bwrap + + if self._local_sandbox_available: + self._user_ns_available = self._check_user_ns_available() + else: + self._user_ns_available = False + + # Set linux32 option + self._linux32 = False + + def create_sandbox(self, *args, **kwargs): + if not self._local_sandbox_available: + return self._create_dummy_sandbox(*args, **kwargs) + else: + return self._create_bwrap_sandbox(*args, **kwargs) + + def check_sandbox_config(self, config): + if not self._local_sandbox_available: + # Accept all sandbox configs as it's irrelevant with the dummy sandbox (no Sandbox.run). + return True + + if self._user_ns_available: + # User namespace support allows arbitrary build UID/GID settings. + pass + elif (config.build_uid != self._uid or config.build_gid != self._gid): + # Without user namespace support, the UID/GID in the sandbox + # will match the host UID/GID. + return False + + # We can't do builds for another host or architecture except x86-32 on + # x86-64 + host_os = self.get_host_os() + host_arch = self.get_host_arch() + if config.build_os != host_os: + raise PlatformError("Configured and host OS don't match.") + elif config.build_arch != host_arch: + # We can use linux32 for building 32bit on 64bit machines + if (host_os == "Linux" and + ((config.build_arch == "x86-32" and host_arch == "x86-64") or + (config.build_arch == "aarch32" and host_arch == "aarch64"))): + # check linux32 is available + try: + utils.get_host_tool('linux32') + self._linux32 = True + except utils.ProgramNotFoundError: + pass + else: + raise PlatformError("Configured architecture and host architecture don't match.") + + return True + + ################################################ + # Private Methods # + ################################################ + + def _create_dummy_sandbox(self, *args, **kwargs): + reasons = [] + if not self._have_fuse: + reasons.append("FUSE is unavailable") + if not self._have_good_bwrap: + if self._bwrap_exists: + reasons.append("`bwrap` is too old (bst needs at least 0.1.2)") + else: + reasons.append("`bwrap` executable not found") + + kwargs['dummy_reason'] = " and ".join(reasons) + return SandboxDummy(*args, **kwargs) + + def _create_bwrap_sandbox(self, *args, **kwargs): + from ..sandbox._sandboxbwrap import SandboxBwrap + # Inform the bubblewrap sandbox as to whether it can use user namespaces or not + kwargs['user_ns_available'] = self._user_ns_available + kwargs['die_with_parent_available'] = self._die_with_parent_available + kwargs['json_status_available'] = self._json_status_available + kwargs['linux32'] = self._linux32 + return SandboxBwrap(*args, **kwargs) + + def _check_user_ns_available(self): + # Here, lets check if bwrap is able to create user namespaces, + # issue a warning if it's not available, and save the state + # locally so that we can inform the sandbox to not try it + # later on. + bwrap = utils.get_host_tool('bwrap') + whoami = utils.get_host_tool('whoami') + try: + output = subprocess.check_output([ + bwrap, + '--ro-bind', '/', '/', + '--unshare-user', + '--uid', '0', '--gid', '0', + whoami, + ], universal_newlines=True).strip() + except subprocess.CalledProcessError: + output = '' + + return output == 'root' diff --git a/src/buildstream/_platform/platform.py b/src/buildstream/_platform/platform.py new file mode 100644 index 000000000..dba60ddca --- /dev/null +++ b/src/buildstream/_platform/platform.py @@ -0,0 +1,164 @@ +# +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . +# +# Authors: +# Tristan Maat + +import os +import platform +import sys +import resource + +from .._exceptions import PlatformError, ImplError + + +class Platform(): + _instance = None + + # Platform() + # + # A class to manage platform-specific details. Currently holds the + # sandbox factory as well as platform helpers. + # + def __init__(self): + self.set_resource_limits() + + @classmethod + def _create_instance(cls): + # Meant for testing purposes and therefore hidden in the + # deepest corners of the source code. Try not to abuse this, + # please? + if os.getenv('BST_FORCE_BACKEND'): + backend = os.getenv('BST_FORCE_BACKEND') + elif sys.platform.startswith('linux'): + backend = 'linux' + elif sys.platform.startswith('darwin'): + backend = 'darwin' + else: + backend = 'unix' + + if backend == 'linux': + from .linux import Linux as PlatformImpl # pylint: disable=cyclic-import + elif backend == 'darwin': + from .darwin import Darwin as PlatformImpl # pylint: disable=cyclic-import + elif backend == 'unix': + from .unix import Unix as PlatformImpl # pylint: disable=cyclic-import + else: + raise PlatformError("No such platform: '{}'".format(backend)) + + cls._instance = PlatformImpl() + + @classmethod + def get_platform(cls): + if not cls._instance: + cls._create_instance() + return cls._instance + + def get_cpu_count(self, cap=None): + cpu_count = len(os.sched_getaffinity(0)) + if cap is None: + return cpu_count + else: + return min(cpu_count, cap) + + @staticmethod + def get_host_os(): + return platform.uname().system + + # canonicalize_arch(): + # + # This returns the canonical, OS-independent architecture name + # or raises a PlatformError if the architecture is unknown. + # + @staticmethod + def canonicalize_arch(arch): + # Note that these are all expected to be lowercase, as we want a + # case-insensitive lookup. Windows can report its arch in ALLCAPS. + aliases = { + "aarch32": "aarch32", + "aarch64": "aarch64", + "aarch64-be": "aarch64-be", + "amd64": "x86-64", + "arm": "aarch32", + "armv8l": "aarch64", + "armv8b": "aarch64-be", + "i386": "x86-32", + "i486": "x86-32", + "i586": "x86-32", + "i686": "x86-32", + "power-isa-be": "power-isa-be", + "power-isa-le": "power-isa-le", + "ppc64": "power-isa-be", + "ppc64le": "power-isa-le", + "sparc": "sparc-v9", + "sparc64": "sparc-v9", + "sparc-v9": "sparc-v9", + "x86-32": "x86-32", + "x86-64": "x86-64" + } + + try: + return aliases[arch.replace('_', '-').lower()] + except KeyError: + raise PlatformError("Unknown architecture: {}".format(arch)) + + # get_host_arch(): + # + # This returns the architecture of the host machine. The possible values + # map from uname -m in order to be a OS independent list. + # + # Returns: + # (string): String representing the architecture + @staticmethod + def get_host_arch(): + # get the hardware identifier from uname + uname_machine = platform.uname().machine + return Platform.canonicalize_arch(uname_machine) + + ################################################################## + # Sandbox functions # + ################################################################## + + # create_sandbox(): + # + # Create a build sandbox suitable for the environment + # + # Args: + # args (dict): The arguments to pass to the sandbox constructor + # kwargs (file): The keyword arguments to pass to the sandbox constructor + # + # Returns: + # (Sandbox) A sandbox + # + def create_sandbox(self, *args, **kwargs): + raise ImplError("Platform {platform} does not implement create_sandbox()" + .format(platform=type(self).__name__)) + + def check_sandbox_config(self, config): + raise ImplError("Platform {platform} does not implement check_sandbox_config()" + .format(platform=type(self).__name__)) + + def set_resource_limits(self, soft_limit=None, hard_limit=None): + # Need to set resources for _frontend/app.py as this is dependent on the platform + # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox. + # Avoid hitting the limit too quickly. + limits = resource.getrlimit(resource.RLIMIT_NOFILE) + if limits[0] != limits[1]: + if soft_limit is None: + soft_limit = limits[1] + if hard_limit is None: + hard_limit = limits[1] + resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit)) diff --git a/src/buildstream/_platform/unix.py b/src/buildstream/_platform/unix.py new file mode 100644 index 000000000..d04b0712c --- /dev/null +++ b/src/buildstream/_platform/unix.py @@ -0,0 +1,56 @@ +# +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . +# +# Authors: +# Tristan Maat + +import os + +from .._exceptions import PlatformError + +from .platform import Platform + + +class Unix(Platform): + + def __init__(self): + + super().__init__() + + self._uid = os.geteuid() + self._gid = os.getegid() + + # Not necessarily 100% reliable, but we want to fail early. + if self._uid != 0: + raise PlatformError("Root privileges are required to run without bubblewrap.") + + def create_sandbox(self, *args, **kwargs): + from ..sandbox._sandboxchroot import SandboxChroot + return SandboxChroot(*args, **kwargs) + + def check_sandbox_config(self, config): + # With the chroot sandbox, the UID/GID in the sandbox + # will match the host UID/GID (typically 0/0). + if config.build_uid != self._uid or config.build_gid != self._gid: + return False + + # Check host os and architecture match + if config.build_os != self.get_host_os(): + raise PlatformError("Configured and host OS don't match.") + elif config.build_arch != self.get_host_arch(): + raise PlatformError("Configured and host architecture don't match.") + + return True -- cgit v1.2.1