Force multiprocessing method to be fork if availablechandan/macos-multiprocessing

Since Python 3.8, the multiprocessing module defaults to the spawn
method on MacOS, since the fork method is considered unsafe.

However, BuildStream is not compatible with spawn mode so we
force the default to fork (if available) which works for the most part.

See
https://lists.apache.org/thread.html/r08edd3dd10ce3039da5eb01f0722d25c44406ab8afadcc3ae78a5414%40%3Cdev.buildstream.apache.org%3E
for some related discussion.

1 files changed, 32 insertions, 0 deletions

diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py
index b0eacbb02..8474d3d81 100644
--- a/src/buildstream/_frontend/cli.py
+++ b/src/buildstream/_frontend/cli.py
@@ -1,3 +1,4 @@
+import multiprocessing
 import os
 import sys
 from functools import partial
@@ -12,6 +13,33 @@ from ..utils import UtilError
 
 
 ##################################################################
+#                   Setup multiprocessing                        #
+##################################################################
+
+# _setup_multiprocessing()
+#
+# Since Python 3.8, the multiprocessing module defaults to
+# the spawn method on MacOS, since the fork method is considered
+# unsafe.
+#
+# However, BuildStream is not compatible with spawn mode so we
+# force the default to fork (if available) which works for the most part.
+#
+# Args:
+#    force (bool): Whether to force our configuration; useful when being called
+#                  from the test suite.
+#
+def _setup_multiprocessing(force=False):
+    # Our tests will not invoke the CLI as a subprocess, but directly from
+    # Python via Cli.run(). So, we allow `force` to be true in this case, but
+    # it is not needed when using the CLI normally.
+    force = "BST_TEST_SUITE" in os.environ
+
+    if multiprocessing.get_start_method() == "spawn" and "fork" in multiprocessing.get_all_start_methods():
+        multiprocessing.set_start_method("fork", force=force)
+
+
+##################################################################
 #              Helper classes and methods for Click              #
 ##################################################################
 
@@ -332,6 +360,10 @@ def cli(context, **kwargs):
     user preferences configuration file.
     """
 
+    # Configuration for multiprocessing module, that we want to do
+    # as early as possible.
+    _setup_multiprocessing()
+
     from .app import App
 
     # Create the App, giving it the main arguments