# # Copyright (C) 2017 Codethink Limited # Copyright (C) 2018 Bloomberg Finance LP # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2 of the License, or (at your option) any later version. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see . # # Authors: # Tristan Maat import multiprocessing import os import platform import sys import psutil from .._exceptions import PlatformError, ImplError, SandboxError from .. import utils class Platform: # Platform() # # A class to manage platform-specific details. Currently holds the # sandbox factory as well as platform helpers. # # Args: # force_sandbox (bool): Force bst to use a particular sandbox # def __init__(self, force_sandbox=None): self.maximize_open_file_limit() self._local_sandbox = None self.dummy_reasons = [] self._setup_sandbox(force_sandbox) def _setup_sandbox(self, force_sandbox): sandbox_setups = {"dummy": self._setup_dummy_sandbox} preferred_sandboxes = [] self._try_sandboxes(force_sandbox, sandbox_setups, preferred_sandboxes) def _try_sandboxes(self, force_sandbox, sandbox_setups, preferred_sandboxes): # Any sandbox from sandbox_setups can be forced by BST_FORCE_SANDBOX # But if a specific sandbox is not forced then only `first class` sandbox are tried before # falling back to the dummy sandbox. # Where `first_class` sandboxes are those in preferred_sandboxes if force_sandbox: try: sandbox_setups[force_sandbox]() except KeyError: raise PlatformError( "Forced Sandbox is unavailable on this platform: BST_FORCE_SANDBOX" " is set to {} but it is not available".format(force_sandbox) ) except SandboxError as Error: raise PlatformError( "Forced Sandbox Error: BST_FORCE_SANDBOX" " is set to {} but cannot be setup".format(force_sandbox), detail=" and ".join(self.dummy_reasons), ) from Error else: for good_sandbox in preferred_sandboxes: try: sandbox_setups[good_sandbox]() return except SandboxError: continue except utils.ProgramNotFoundError: continue sandbox_setups["dummy"]() def _check_sandbox(self, Sandbox): try: Sandbox.check_available() except SandboxError as Error: self.dummy_reasons += Sandbox._dummy_reasons raise Error @classmethod def create_instance(cls): # Meant for testing purposes and therefore hidden in the # deepest corners of the source code. Try not to abuse this, # please? if os.getenv("BST_FORCE_SANDBOX"): force_sandbox = os.getenv("BST_FORCE_SANDBOX") else: force_sandbox = None if os.getenv("BST_FORCE_BACKEND"): backend = os.getenv("BST_FORCE_BACKEND") elif sys.platform.startswith("darwin"): backend = "darwin" elif sys.platform.startswith("linux"): backend = "linux" elif sys.platform == "win32": backend = "win32" else: backend = "fallback" if backend == "linux": from .linux import Linux as PlatformImpl # pylint: disable=cyclic-import elif backend == "darwin": from .darwin import Darwin as PlatformImpl # pylint: disable=cyclic-import elif backend == "win32": from .win32 import Win32 as PlatformImpl # pylint: disable=cyclic-import elif backend == "fallback": from .fallback import Fallback as PlatformImpl # pylint: disable=cyclic-import else: raise PlatformError("No such platform: '{}'".format(backend)) return PlatformImpl(force_sandbox=force_sandbox) def get_cpu_count(self, cap=None): cpu_count = len(psutil.Process().cpu_affinity()) if cap is None: return cpu_count else: return min(cpu_count, cap) @staticmethod def get_host_os(): return platform.uname().system # canonicalize_arch(): # # This returns the canonical, OS-independent architecture name # or raises a PlatformError if the architecture is unknown. # @staticmethod def canonicalize_arch(arch): # Note that these are all expected to be lowercase, as we want a # case-insensitive lookup. Windows can report its arch in ALLCAPS. aliases = { "aarch32": "aarch32", "aarch64": "aarch64", "aarch64-be": "aarch64-be", "amd64": "x86-64", "arm": "aarch32", "armv8l": "aarch64", "armv8b": "aarch64-be", "i386": "x86-32", "i486": "x86-32", "i586": "x86-32", "i686": "x86-32", "power-isa-be": "power-isa-be", "power-isa-le": "power-isa-le", "ppc64": "power-isa-be", "ppc64le": "power-isa-le", "sparc": "sparc-v9", "sparc64": "sparc-v9", "sparc-v9": "sparc-v9", "x86-32": "x86-32", "x86-64": "x86-64", } try: return aliases[arch.replace("_", "-").lower()] except KeyError: raise PlatformError("Unknown architecture: {}".format(arch)) # get_host_arch(): # # This returns the architecture of the host machine. The possible values # map from uname -m in order to be a OS independent list. # # Returns: # (string): String representing the architecture @staticmethod def get_host_arch(): # get the hardware identifier from uname uname_machine = platform.uname().machine return Platform.canonicalize_arch(uname_machine) # does_multiprocessing_start_require_pickling(): # # Returns True if the multiprocessing start method will pickle arguments # to new processes. # # Returns: # (bool): Whether pickling is required or not # def does_multiprocessing_start_require_pickling(self): # Note that if the start method has not been set before now, it will be # set to the platform default by `get_start_method`. return multiprocessing.get_start_method() != "fork" ################################################################## # Sandbox functions # ################################################################## # create_sandbox(): # # Create a build sandbox suitable for the environment # # Args: # args (dict): The arguments to pass to the sandbox constructor # kwargs (file): The keyword arguments to pass to the sandbox constructor # # Returns: # (Sandbox) A sandbox # def create_sandbox(self, *args, **kwargs): raise ImplError("Platform {platform} does not implement create_sandbox()".format(platform=type(self).__name__)) def check_sandbox_config(self, config): raise ImplError( "Platform {platform} does not implement check_sandbox_config()".format(platform=type(self).__name__) ) def maximize_open_file_limit(self): # Need to set resources for _frontend/app.py as this is dependent on the platform # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox. # Avoid hitting the limit too quickly, by increasing it as far as we can. # Import this late, as it is not available on Windows. Note that we # could use `psutil.Process().rlimit` instead, but this would introduce # a dependency on the `prlimit(2)` call, which seems to only be # available on Linux. For more info: # https://github.com/giampaolo/psutil/blob/cbf2bafbd33ad21ef63400d94cb313c299e78a45/psutil/_psutil_linux.c#L45 import resource soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) if soft_limit != hard_limit: resource.setrlimit(resource.RLIMIT_NOFILE, (hard_limit, hard_limit)) def _setup_dummy_sandbox(self): raise ImplError( "Platform {platform} does not implement _setup_dummy_sandbox()".format(platform=type(self).__name__) )