Merge branch 'aevri/enable_spawn_ci_5a' into 'master'

spawn tests: enable most non-integration

See merge request BuildStream/buildstream!1676

3 files changed, 40 insertions, 19 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 21de1f2de..f669e1905 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -199,7 +199,7 @@ tests-spawn-multiprocessing-start-method:
     - mkdir -p "${INTEGRATION_CACHE}"
     - useradd -Um buildstream
     - chown -R buildstream:buildstream .
-    - su buildstream -c "tox --  ${PYTEST_ARGS} tests/{cachekey,elements,format,plugins,internals,sourcecache}"
+    - su buildstream -c "tox -- ${PYTEST_ARGS} tests/{artifactcache,cachekey,elements,format,frontend,internals,plugins,sourcecache}"
 
 # Run type checkers
 mypy:
diff --git a/src/buildstream/_plugincontext.py b/src/buildstream/_plugincontext.py
index 9e32f4992..b07c2b31a 100644
--- a/src/buildstream/_plugincontext.py
+++ b/src/buildstream/_plugincontext.py
@@ -46,6 +46,12 @@ class PluginContext():
     def __init__(self, plugin_base, base_type, site_plugin_path, *,
                  plugin_origins=None, format_versions={}):
 
+        # For pickling across processes, make sure this context has a unique
+        # identifier, which we prepend to the identifier of each PluginSource.
+        # This keeps plugins loaded during the first and second pass distinct
+        # from eachother.
+        self._identifier = str(id(self))
+
         # The plugin kinds which were loaded
         self.loaded_dependencies = []
 
@@ -59,12 +65,17 @@ class PluginContext():
         # The PluginSource object
         self._plugin_base = plugin_base
         self._site_plugin_path = site_plugin_path
-        self._site_source = plugin_base.make_plugin_source(
-            searchpath=self._site_plugin_path,
-        )
         self._alternate_sources = {}
         self._format_versions = format_versions
 
+        self._init_site_source()
+
+    def _init_site_source(self):
+        self._site_source = self._plugin_base.make_plugin_source(
+            searchpath=self._site_plugin_path,
+            identifier=self._identifier + 'site',
+        )
+
     def __getstate__(self):
         state = self.__dict__.copy()
 
@@ -98,9 +109,7 @@ class PluginContext():
         # of the PluginSource. We would also have to recreate `_types` as it
         # was before unpickling them. We are not using this method in
         # BuildStream, so the identifier is not restored here.
-        self._site_source = self._plugin_base.make_plugin_source(
-            searchpath=self._site_plugin_path,
-        )
+        self._init_site_source()
 
     # lookup():
     #
@@ -126,7 +135,10 @@ class PluginContext():
     def _get_local_plugin_source(self, path):
         if ('local', path) not in self._alternate_sources:
             # key by a tuple to avoid collision
-            source = self._plugin_base.make_plugin_source(searchpath=[path])
+            source = self._plugin_base.make_plugin_source(
+                searchpath=[path],
+                identifier=self._identifier + path,
+            )
             # Ensure that sources never get garbage collected,
             # as they'll take the plugins with them.
             self._alternate_sources[('local', path)] = source
@@ -167,7 +179,10 @@ class PluginContext():
                 # The plugin didn't have an accompanying YAML file
                 defaults = None
 
-            source = self._plugin_base.make_plugin_source(searchpath=[os.path.dirname(location)])
+            source = self._plugin_base.make_plugin_source(
+                searchpath=[os.path.dirname(location)],
+                identifier=self._identifier + os.path.dirname(location),
+            )
             self._alternate_sources[('pip', package_name)] = source
 
         else:
diff --git a/tests/sandboxes/missing_dependencies.py b/tests/sandboxes/missing_dependencies.py
index 975c8eb00..a5bf31e76 100644
--- a/tests/sandboxes/missing_dependencies.py
+++ b/tests/sandboxes/missing_dependencies.py
@@ -18,13 +18,19 @@ DATA_DIR = os.path.join(
 )
 
 
+def _symlink_host_tools_to_dir(host_tools, dir_):
+    dir_.mkdir(exist_ok=True)
+    for tool in host_tools:
+        target_path = dir_ / tool
+        os.symlink(utils.get_host_tool(tool), str(target_path))
+
+
 @pytest.mark.skipif(not IS_LINUX, reason='Only available on Linux')
 @pytest.mark.datafiles(DATA_DIR)
-def test_missing_brwap_has_nice_error_message(cli, datafiles, tmp_path):
-    # Create symlink to buildbox-casd to work with custom PATH
-    buildbox_casd = tmp_path.joinpath('bin/buildbox-casd')
-    buildbox_casd.parent.mkdir()
-    os.symlink(utils.get_host_tool('buildbox-casd'), str(buildbox_casd))
+def test_missing_bwrap_has_nice_error_message(cli, datafiles, tmp_path):
+    # Create symlink to buildbox-casd and git to work with custom PATH
+    bin_dir = tmp_path / "bin"
+    _symlink_host_tools_to_dir(['buildbox-casd', 'git'], bin_dir)
 
     project = str(datafiles)
     element_path = os.path.join(project, 'elements', 'element.bst')
@@ -50,7 +56,7 @@ def test_missing_brwap_has_nice_error_message(cli, datafiles, tmp_path):
     result = cli.run(
         project=project,
         args=['build', 'element.bst'],
-        env={'PATH': str(tmp_path.joinpath('bin')),
+        env={'PATH': str(bin_dir),
              'BST_FORCE_SANDBOX': None})
     result.assert_task_error(ErrorDomain.SANDBOX, 'unavailable-local-sandbox')
     assert "not found" in result.stderr
@@ -69,9 +75,9 @@ def test_old_brwap_has_nice_error_message(cli, datafiles, tmp_path):
 
     bwrap.chmod(0o755)
 
-    # Create symlink to buildbox-casd to work with custom PATH
-    buildbox_casd = tmp_path.joinpath('bin/buildbox-casd')
-    os.symlink(utils.get_host_tool('buildbox-casd'), str(buildbox_casd))
+    # Create symlink to buildbox-casd and git to work with custom PATH
+    bin_dir = tmp_path / "bin"
+    _symlink_host_tools_to_dir(['buildbox-casd', 'git'], bin_dir)
 
     project = str(datafiles)
     element_path = os.path.join(project, 'elements', 'element3.bst')
@@ -97,7 +103,7 @@ def test_old_brwap_has_nice_error_message(cli, datafiles, tmp_path):
     result = cli.run(
         project=project,
         args=['--debug', '--verbose', 'build', 'element3.bst'],
-        env={'PATH': str(tmp_path.joinpath('bin')),
+        env={'PATH': str(bin_dir),
              'BST_FORCE_SANDBOX': None})
     result.assert_task_error(ErrorDomain.SANDBOX, 'unavailable-local-sandbox')
     assert "too old" in result.stderr