From daf0bcf6da4c432ccf38d74c2097bc45b4ede485 Mon Sep 17 00:00:00 2001 From: Angelos Evripiotis Date: Thu, 17 Oct 2019 12:15:35 +0100 Subject: tox.ini: pass through BST_FORCE_START_METHOD Otherwise we won't actually be testing the spawn code path. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 5ea15f50d..789aa40bf 100644 --- a/tox.ini +++ b/tox.ini @@ -38,6 +38,7 @@ passenv = ARTIFACT_CACHE_SERVICE BST_FORCE_BACKEND BST_FORCE_SANDBOX + BST_FORCE_START_METHOD GI_TYPELIB_PATH INTEGRATION_CACHE http_proxy -- cgit v1.2.1 From 702dddae4197b43a93ec194f88457708a42119dd Mon Sep 17 00:00:00 2001 From: Angelos Evripiotis Date: Thu, 17 Oct 2019 12:12:47 +0100 Subject: tests/conftest: implement BST_FORCE_START_METHOD --- tests/conftest.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 68fdba7fb..4aa7be7fb 100755 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,6 +20,7 @@ # Tristan Maat # import os +import multiprocessing import pytest from buildstream.testing import register_repo_kind, sourcetests_collection_hook @@ -155,3 +156,17 @@ def set_xdg_paths(pytestconfig): value = os.path.join(pytestconfig.getoption("basetemp"), default) os.environ[env_var] = value + + +def pytest_configure(config): + # If we need to set_start_method() then we need to do it as early as + # possible. Note that some tests implicitly set the start method by using + # multiprocessing. If we wait for bst to do it, it will already be too + # late. + print( + "Multiprocessing method:", + multiprocessing.get_start_method(allow_none=True), + ) + if 'BST_FORCE_START_METHOD' in os.environ: + start_method = os.environ['BST_FORCE_START_METHOD'] + multiprocessing.set_start_method(start_method) -- cgit v1.2.1 From 11cd08206eb3c4bbb5d82d6a71f7773997ded6a9 Mon Sep 17 00:00:00 2001 From: Angelos Evripiotis Date: Thu, 17 Oct 2019 12:10:55 +0100 Subject: cli: BST_FORCE_START_METHOD only sets if necessary Allow situations where the start method is already set, this enables us to use this in testing situations. Also, print a diagnostic if it's already set to something we didn't want. Now this block got more complex, split out into a new function. Now we're using this string a lot, extract it to a variable, to make sure we're spelling it correctly everywhere. --- src/buildstream/_frontend/cli.py | 49 ++++++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index 60527e698..42d557615 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -202,6 +202,41 @@ def validate_output_streams(): sys.exit(-1) +def handle_bst_force_start_method_env(): + bst_force_start_method_str = "BST_FORCE_START_METHOD" + if bst_force_start_method_str in os.environ: + start_method = os.environ[bst_force_start_method_str] + existing_start_method = multiprocessing.get_start_method(allow_none=True) + if existing_start_method is None: + multiprocessing.set_start_method(start_method) + print( + bst_force_start_method_str + ": multiprocessing start method forced to:", + start_method, + file=sys.stderr, + flush=True, + ) + elif existing_start_method == start_method: + # Note that when testing, we run the buildstream entrypoint + # multiple times in the same executable, so guard against that + # here. + print( + bst_force_start_method_str + ": multiprocessing start method already set to:", + existing_start_method, + file=sys.stderr, + flush=True, + ) + else: + print( + bst_force_start_method_str + ": cannot set multiprocessing start method to:", + start_method, + ", already set to:", + existing_start_method, + file=sys.stderr, + flush=True, + ) + sys.exit(-1) + + def override_main(self, args=None, prog_name=None, complete_var=None, standalone_mode=True, **extra): @@ -229,16 +264,10 @@ def override_main(self, args=None, prog_name=None, complete_var=None, validate_output_streams() # We can only set the global multiprocessing start method once; for that - # reason we're advised to do it inside the entrypoint, where it is easy to - # ensure the code path is only followed once. - if 'BST_FORCE_START_METHOD' in os.environ: - multiprocessing.set_start_method(os.environ['BST_FORCE_START_METHOD']) - print( - "BST_FORCE_START_METHOD: multiprocessing start method forced to:", - os.environ['BST_FORCE_START_METHOD'], - file=sys.stderr, - flush=True, - ) + # reason we're advised to do it inside the entrypoint, where it's more + # likely that you can ensure the code path is only followed once. In the + # case of testing, our tests preceed our entrypoint, so we do our best. + handle_bst_force_start_method_env() original_main(self, args=args, prog_name=prog_name, complete_var=None, standalone_mode=standalone_mode, **extra) -- cgit v1.2.1 From ac8ce6053846023b154b70905139884c6c45bb68 Mon Sep 17 00:00:00 2001 From: Angelos Evripiotis Date: Thu, 17 Oct 2019 12:05:42 +0100 Subject: .gitlab-ci.yml: disable most spawn tests --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bda1be87f..96cc4b2b1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -194,6 +194,12 @@ tests-spawn-multiprocessing-start-method: <<: *tests variables: BST_FORCE_START_METHOD: "spawn" + script: + # FIXME: Until all the tests pass as normal, override which tests will run here. + - mkdir -p "${INTEGRATION_CACHE}" + - useradd -Um buildstream + - chown -R buildstream:buildstream . + - su buildstream -c "tox -- ${PYTEST_ARGS} tests/{cachekey,plugins}" # Run type checkers mypy: -- cgit v1.2.1