Merge branch 'mac_fixes' into 'master'

Implement compatibility fixes for MacOSX and WSL Blocks #411 and #412"

See merge request BuildStream/buildstream!726

9 files changed, 158 insertions, 44 deletions

diff --git a/buildstream/_frontend/app.py b/buildstream/_frontend/app.py
index a1afee478..f3dcd623b 100644
--- a/buildstream/_frontend/app.py
+++ b/buildstream/_frontend/app.py
@@ -115,14 +115,6 @@ class App():
         else:
             self.colors = False
 
-        # Increase the soft limit for open file descriptors to the maximum.
-        # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
-        # Avoid hitting the limit too quickly.
-        limits = resource.getrlimit(resource.RLIMIT_NOFILE)
-        if limits[0] != limits[1]:
-            # Set soft limit to hard limit
-            resource.setrlimit(resource.RLIMIT_NOFILE, (limits[1], limits[1]))
-
     # create()
     #
     # Should be used instead of the regular constructor.
diff --git a/buildstream/_platform/darwin.py b/buildstream/_platform/darwin.py
new file mode 100644
index 000000000..7092eb2aa
--- /dev/null
+++ b/buildstream/_platform/darwin.py
@@ -0,0 +1,50 @@
+#
+#  Copyright (C) 2017 Codethink Limited
+#  Copyright (C) 2018 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import resource
+
+from .._exceptions import PlatformError
+from ..sandbox import SandboxDummy
+
+from . import Platform
+
+
+class Darwin(Platform):
+
+    # This value comes from OPEN_MAX in syslimits.h
+    OPEN_MAX = 10240
+
+    def __init__(self):
+
+        super().__init__()
+
+    def create_sandbox(self, *args, **kwargs):
+        return SandboxDummy(*args, **kwargs)
+
+    def check_sandbox_config(self, config):
+        # Accept all sandbox configs as it's irrelevant with the dummy sandbox (no Sandbox.run).
+        return True
+
+    def get_cpu_count(self, cap=None):
+        if cap < os.cpu_count():
+            return cap
+        else:
+            return os.cpu_count()
+
+    def set_resource_limits(self, soft_limit=OPEN_MAX, hard_limit=None):
+        super().set_resource_limits(soft_limit)
diff --git a/buildstream/_platform/linux.py b/buildstream/_platform/linux.py
index 7af1a2283..85bfbd852 100644
--- a/buildstream/_platform/linux.py
+++ b/buildstream/_platform/linux.py
@@ -23,7 +23,7 @@ import subprocess
 from .. import _site
 from .. import utils
 from .._message import Message, MessageType
-from ..sandbox import SandboxBwrap
+from ..sandbox import SandboxDummy
 
 from . import Platform
 
@@ -38,13 +38,21 @@ class Linux(Platform):
         self._gid = os.getegid()
 
         self._die_with_parent_available = _site.check_bwrap_version(0, 1, 8)
-        self._user_ns_available = self._check_user_ns_available()
+
+        if self._local_sandbox_available():
+            self._user_ns_available = self._check_user_ns_available()
+        else:
+            self._user_ns_available = False
 
     def create_sandbox(self, *args, **kwargs):
-        # Inform the bubblewrap sandbox as to whether it can use user namespaces or not
-        kwargs['user_ns_available'] = self._user_ns_available
-        kwargs['die_with_parent_available'] = self._die_with_parent_available
-        return SandboxBwrap(*args, **kwargs)
+        if not self._local_sandbox_available():
+            return SandboxDummy(*args, **kwargs)
+        else:
+            from ..sandbox._sandboxbwrap import SandboxBwrap
+            # Inform the bubblewrap sandbox as to whether it can use user namespaces or not
+            kwargs['user_ns_available'] = self._user_ns_available
+            kwargs['die_with_parent_available'] = self._die_with_parent_available
+            return SandboxBwrap(*args, **kwargs)
 
     def check_sandbox_config(self, config):
         if self._user_ns_available:
@@ -58,8 +66,13 @@ class Linux(Platform):
     ################################################
     #              Private Methods                 #
     ################################################
-    def _check_user_ns_available(self):
+    def _local_sandbox_available(self):
+        try:
+            return os.path.exists(utils.get_host_tool('bwrap')) and os.path.exists('/dev/fuse')
+        except utils.ProgramNotFoundError:
+            return False
 
+    def _check_user_ns_available(self):
         # Here, lets check if bwrap is able to create user namespaces,
         # issue a warning if it's not available, and save the state
         # locally so that we can inform the sandbox to not try it
diff --git a/buildstream/_platform/platform.py b/buildstream/_platform/platform.py
index b37964986..bc6a624c4 100644
--- a/buildstream/_platform/platform.py
+++ b/buildstream/_platform/platform.py
@@ -19,6 +19,7 @@
 
 import os
 import sys
+import resource
 
 from .._exceptions import PlatformError, ImplError
 
@@ -32,23 +33,26 @@ class Platform():
     # sandbox factory as well as platform helpers.
     #
     def __init__(self):
-        pass
+        self.set_resource_limits()
 
     @classmethod
     def _create_instance(cls):
-        if sys.platform.startswith('linux'):
-            backend = 'linux'
-        else:
-            backend = 'unix'
-
         # Meant for testing purposes and therefore hidden in the
         # deepest corners of the source code. Try not to abuse this,
         # please?
         if os.getenv('BST_FORCE_BACKEND'):
             backend = os.getenv('BST_FORCE_BACKEND')
+        elif sys.platform.startswith('linux'):
+            backend = 'linux'
+        elif sys.platform.startswith('darwin'):
+            backend = 'darwin'
+        else:
+            backend = 'unix'
 
         if backend == 'linux':
             from .linux import Linux as PlatformImpl
+        elif backend == 'darwin':
+            from .darwin import Darwin as PlatformImpl
         elif backend == 'unix':
             from .unix import Unix as PlatformImpl
         else:
@@ -62,6 +66,9 @@ class Platform():
             cls._create_instance()
         return cls._instance
 
+    def get_cpu_count(self, cap=None):
+        return min(len(os.sched_getaffinity(0)), cap)
+
     ##################################################################
     #                        Sandbox functions                       #
     ##################################################################
@@ -84,3 +91,15 @@ class Platform():
     def check_sandbox_config(self, config):
         raise ImplError("Platform {platform} does not implement check_sandbox_config()"
                         .format(platform=type(self).__name__))
+
+    def set_resource_limits(self, soft_limit=None, hard_limit=None):
+        # Need to set resources for _frontend/app.py as this is dependent on the platform
+        # SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
+        # Avoid hitting the limit too quickly.
+        limits = resource.getrlimit(resource.RLIMIT_NOFILE)
+        if limits[0] != limits[1]:
+            if soft_limit is None:
+                soft_limit = limits[1]
+            if hard_limit is None:
+                hard_limit = limits[1]
+            resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit))
diff --git a/buildstream/_platform/unix.py b/buildstream/_platform/unix.py
index 7aa8cbc0d..d2acefe65 100644
--- a/buildstream/_platform/unix.py
+++ b/buildstream/_platform/unix.py
@@ -20,7 +20,6 @@
 import os
 
 from .._exceptions import PlatformError
-from ..sandbox import SandboxChroot
 
 from . import Platform
 
@@ -39,6 +38,7 @@ class Unix(Platform):
             raise PlatformError("Root privileges are required to run without bubblewrap.")
 
     def create_sandbox(self, *args, **kwargs):
+        from ..sandbox._sandboxchroot import SandboxChroot
         return SandboxChroot(*args, **kwargs)
 
     def check_sandbox_config(self, config):
diff --git a/buildstream/_project.py b/buildstream/_project.py
index b72318af9..44e5171e6 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -38,6 +38,7 @@ from ._loader import Loader
 from .element import Element
 from ._message import Message, MessageType
 from ._includes import Includes
+from ._platform import Platform
 
 
 # Project Configuration file
@@ -617,7 +618,8 @@ class Project():
         # Based on some testing (mainly on AWS), maximum effective
         # max-jobs value seems to be around 8-10 if we have enough cores
         # users should set values based on workload and build infrastructure
-        output.base_variables['max-jobs'] = str(min(len(os.sched_getaffinity(0)), 8))
+        platform = Platform.get_platform()
+        output.base_variables['max-jobs'] = str(platform.get_cpu_count(8))
 
         # Export options into variables, if that was requested
         output.options.export_variables(output.base_variables)
diff --git a/buildstream/sandbox/__init__.py b/buildstream/sandbox/__init__.py
index 2c76e9e8e..5999aba7a 100644
--- a/buildstream/sandbox/__init__.py
+++ b/buildstream/sandbox/__init__.py
@@ -18,6 +18,5 @@
 #        Tristan Maat <tristan.maat@codethink.co.uk>
 
 from .sandbox import Sandbox, SandboxFlags
-from ._sandboxchroot import SandboxChroot
-from ._sandboxbwrap import SandboxBwrap
 from ._sandboxremote import SandboxRemote
+from ._sandboxdummy import SandboxDummy
diff --git a/buildstream/sandbox/_sandboxdummy.py b/buildstream/sandbox/_sandboxdummy.py
new file mode 100644
index 000000000..51239a4ea
--- /dev/null
+++ b/buildstream/sandbox/_sandboxdummy.py
@@ -0,0 +1,40 @@
+#
+#  Copyright (C) 2017 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+
+from .._exceptions import SandboxError
+from . import Sandbox
+
+
+class SandboxDummy(Sandbox):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def run(self, command, flags, *, cwd=None, env=None):
+
+        # Fallback to the sandbox default settings for
+        # the cwd and env.
+        #
+        cwd = self._get_work_directory(cwd=cwd)
+        env = self._get_environment(cwd=cwd, env=env)
+
+        if not self._has_command(command[0], env):
+            raise SandboxError("Staged artifacts do not provide command "
+                               "'{}'".format(command[0]),
+                               reason='missing-command')
+
+        raise SandboxError("This platform does not support local builds")
diff --git a/buildstream/utils.py b/buildstream/utils.py
index 60211f35b..1e04a31ed 100644
--- a/buildstream/utils.py
+++ b/buildstream/utils.py
@@ -35,6 +35,7 @@ import tempfile
 import itertools
 import functools
 from contextlib import contextmanager
+from stat import S_ISDIR
 
 import psutil
 
@@ -328,27 +329,25 @@ def safe_remove(path):
     Raises:
        UtilError: In the case of unexpected system call failures
     """
-    if os.path.lexists(path):
-
-        # Try to remove anything that is in the way, but issue
-        # a warning instead if it removes a non empty directory
-        try:
+    try:
+        if S_ISDIR(os.lstat(path).st_mode):
+            os.rmdir(path)
+        else:
             os.unlink(path)
-        except OSError as e:
-            if e.errno != errno.EISDIR:
-                raise UtilError("Failed to remove '{}': {}"
-                                .format(path, e))
-
-            try:
-                os.rmdir(path)
-            except OSError as e:
-                if e.errno == errno.ENOTEMPTY:
-                    return False
-                else:
-                    raise UtilError("Failed to remove '{}': {}"
-                                    .format(path, e))
 
-    return True
+        # File removed/unlinked successfully
+        return True
+
+    except OSError as e:
+        if e.errno == errno.ENOTEMPTY:
+            # Path is non-empty directory
+            return False
+        elif e.errno == errno.ENOENT:
+            # Path does not exist
+            return True
+
+        raise UtilError("Failed to remove '{}': {}"
+                        .format(path, e))
 
 
 def copy_files(src, dest, *, files=None, ignore_missing=False, report_written=False):