Merge branch 'aevri/enable_spawn_ci' into 'master'

Enable a subset of `tests-spawn-multiprocessing-start-method`

See merge request BuildStream/buildstream!1650

4 files changed, 61 insertions, 10 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bda1be87f..96cc4b2b1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -194,6 +194,12 @@ tests-spawn-multiprocessing-start-method:
   <<: *tests
   variables:
     BST_FORCE_START_METHOD: "spawn"
+  script:
+    # FIXME: Until all the tests pass as normal, override which tests will run here.
+    - mkdir -p "${INTEGRATION_CACHE}"
+    - useradd -Um buildstream
+    - chown -R buildstream:buildstream .
+    - su buildstream -c "tox --  ${PYTEST_ARGS} tests/{cachekey,plugins}"
 
 # Run type checkers
 mypy:
diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py
index 60527e698..42d557615 100644
--- a/src/buildstream/_frontend/cli.py
+++ b/src/buildstream/_frontend/cli.py
@@ -202,6 +202,41 @@ def validate_output_streams():
             sys.exit(-1)
 
 
+def handle_bst_force_start_method_env():
+    bst_force_start_method_str = "BST_FORCE_START_METHOD"
+    if bst_force_start_method_str in os.environ:
+        start_method = os.environ[bst_force_start_method_str]
+        existing_start_method = multiprocessing.get_start_method(allow_none=True)
+        if existing_start_method is None:
+            multiprocessing.set_start_method(start_method)
+            print(
+                bst_force_start_method_str + ": multiprocessing start method forced to:",
+                start_method,
+                file=sys.stderr,
+                flush=True,
+            )
+        elif existing_start_method == start_method:
+            # Note that when testing, we run the buildstream entrypoint
+            # multiple times in the same executable, so guard against that
+            # here.
+            print(
+                bst_force_start_method_str + ": multiprocessing start method already set to:",
+                existing_start_method,
+                file=sys.stderr,
+                flush=True,
+            )
+        else:
+            print(
+                bst_force_start_method_str + ": cannot set multiprocessing start method to:",
+                start_method,
+                ", already set to:",
+                existing_start_method,
+                file=sys.stderr,
+                flush=True,
+            )
+            sys.exit(-1)
+
+
 def override_main(self, args=None, prog_name=None, complete_var=None,
                   standalone_mode=True, **extra):
 
@@ -229,16 +264,10 @@ def override_main(self, args=None, prog_name=None, complete_var=None,
     validate_output_streams()
 
     # We can only set the global multiprocessing start method once; for that
-    # reason we're advised to do it inside the entrypoint, where it is easy to
-    # ensure the code path is only followed once.
-    if 'BST_FORCE_START_METHOD' in os.environ:
-        multiprocessing.set_start_method(os.environ['BST_FORCE_START_METHOD'])
-        print(
-            "BST_FORCE_START_METHOD: multiprocessing start method forced to:",
-            os.environ['BST_FORCE_START_METHOD'],
-            file=sys.stderr,
-            flush=True,
-        )
+    # reason we're advised to do it inside the entrypoint, where it's more
+    # likely that you can ensure the code path is only followed once. In the
+    # case of testing, our tests preceed our entrypoint, so we do our best.
+    handle_bst_force_start_method_env()
 
     original_main(self, args=args, prog_name=prog_name, complete_var=None,
                   standalone_mode=standalone_mode, **extra)
diff --git a/tests/conftest.py b/tests/conftest.py
index 68fdba7fb..4aa7be7fb 100755
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -20,6 +20,7 @@
 #        Tristan Maat <tristan.maat@codethink.co.uk>
 #
 import os
+import multiprocessing
 import pytest
 
 from buildstream.testing import register_repo_kind, sourcetests_collection_hook
@@ -155,3 +156,17 @@ def set_xdg_paths(pytestconfig):
             value = os.path.join(pytestconfig.getoption("basetemp"), default)
 
         os.environ[env_var] = value
+
+
+def pytest_configure(config):
+    # If we need to set_start_method() then we need to do it as early as
+    # possible. Note that some tests implicitly set the start method by using
+    # multiprocessing. If we wait for bst to do it, it will already be too
+    # late.
+    print(
+        "Multiprocessing method:",
+        multiprocessing.get_start_method(allow_none=True),
+    )
+    if 'BST_FORCE_START_METHOD' in os.environ:
+        start_method = os.environ['BST_FORCE_START_METHOD']
+        multiprocessing.set_start_method(start_method)
diff --git a/tox.ini b/tox.ini
index 5ea15f50d..789aa40bf 100644
--- a/tox.ini
+++ b/tox.ini
@@ -38,6 +38,7 @@ passenv =
     ARTIFACT_CACHE_SERVICE
     BST_FORCE_BACKEND
     BST_FORCE_SANDBOX
+    BST_FORCE_START_METHOD
     GI_TYPELIB_PATH
     INTEGRATION_CACHE
     http_proxy