_project.py: Implement internal junctions

This commit adds support for the `internal` list of the project's
`junctions` configuration section.

_loader/loadcontext.py is updated with this commit to consider
the internal markers and avoid raising errors for collisions with
internal junctions, as well as being enhanced to list internal
junctions in any conflicting junction error messages.

2 files changed, 77 insertions, 13 deletions

diff --git a/src/buildstream/_loader/loadcontext.py b/src/buildstream/_loader/loadcontext.py
index c6d256050..6183a192b 100644
--- a/src/buildstream/_loader/loadcontext.py
+++ b/src/buildstream/_loader/loadcontext.py
@@ -55,25 +55,29 @@ class ProjectLoaders:
     #
     def assert_loaders(self):
         duplicates = {}
+        internal = {}
         primary = []
 
         for loader in self._collect:
-            duplicating = self._search_duplicates(loader)
+            duplicating, internalizing = self._search_project_relationships(loader)
             if duplicating:
                 duplicates[loader] = duplicating
-            else:
+            if internalizing:
+                internal[loader] = internalizing
+
+            if not (duplicating or internalizing):
                 primary.append(loader)
 
         if len(primary) > 1:
-            self._raise_conflict(duplicates)
+            self._raise_conflict(duplicates, internal)
 
         elif primary and duplicates:
-            self._raise_conflict(duplicates)
+            self._raise_conflict(duplicates, internal)
 
-    # _search_duplicates()
+    # _search_project_relationships()
     #
     # Searches this loader's ancestry for projects which mark this
-    # loader as a duplicate.
+    # loader as internal or duplicate
     #
     # Args:
     #    loader (Loader): The loader to search for duplicate markers of
@@ -81,13 +85,18 @@ class ProjectLoaders:
     # Returns:
     #    (list): A list of Loader objects who's project has marked
     #            this junction as a duplicate
+    #    (list): A list of Loader objects who's project has marked
+    #            this junction as internal
     #
-    def _search_duplicates(self, loader):
+    def _search_project_relationships(self, loader):
         duplicates = []
+        internal = []
         for parent in loader.ancestors():
             if parent.project.junction_is_duplicated(self._name, loader):
                 duplicates.append(parent)
-        return duplicates
+            if parent.project.junction_is_internal(loader):
+                internal.append(parent)
+        return duplicates, internal
 
     # _raise_conflict()
     #
@@ -98,16 +107,24 @@ class ProjectLoaders:
     # Args:
     #    duplicates (dict): A table of duplicating Loaders, indexed
     #                       by duplicated Loader
+    #    internals (dict): A table of Loaders which mark a loader as internal,
+    #                      indexed by internal Loader
     #
     # Raises:
     #    (LoadError): In case there is a CONFLICTING_JUNCTION error
     #
-    def _raise_conflict(self, duplicates):
-        lines = [self._loader_description(loader, duplicates) for loader in self._collect]
+    def _raise_conflict(self, duplicates, internals):
+        explanation = (
+            "Internal projects do not cause any conflicts. Conflicts can also be avoided\n"
+            + "by marking every instance of the project as a duplicate."
+        )
+        lines = [self._loader_description(loader, duplicates, internals) for loader in self._collect]
+        detail = "{}\n{}".format("\n".join(lines), explanation)
+
         raise LoadError(
             "Project '{}' was loaded in multiple contexts".format(self._name),
             LoadErrorReason.CONFLICTING_JUNCTION,
-            detail="\n".join(lines),
+            detail=detail,
         )
 
     # _loader_description()
@@ -116,11 +133,13 @@ class ProjectLoaders:
     #    loader (Loader): The loader to describe
     #    duplicates (dict): A table of duplicating Loaders, indexed
     #                       by duplicated Loader
+    #    internals (dict): A table of Loaders which mark a loader as internal,
+    #                      indexed by internal Loader
     #
     # Returns:
     #    (str): A string representing how this loader was loaded
     #
-    def _loader_description(self, loader, duplicates):
+    def _loader_description(self, loader, duplicates, internals):
 
         line = "{}\n".format(loader)
 
@@ -130,6 +149,12 @@ class ProjectLoaders:
             for dup in duplicating:
                 line += "  Duplicated by: {}\n".format(dup)
 
+        # Mention projects which have marked this project as internal
+        internalizing = internals.get(loader)
+        if internalizing:
+            for internal in internalizing:
+                line += "  Internal to: {}\n".format(internal)
+
         return line
 
 
diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py
index 3a8998abb..21dc2b9d2 100644
--- a/src/buildstream/_project.py
+++ b/src/buildstream/_project.py
@@ -163,6 +163,10 @@ class Project:
         # provenances as values
         self._junction_duplicates = {}
 
+        # A table of project relative junctions to consider as 'internal'. The values
+        # of the table are simply used to store ProvenanceInformation.
+        self._junction_internal = {}
+
         self._context.add_project(self)
 
         self._partially_loaded = False
@@ -583,6 +587,34 @@ class Project:
 
         return False
 
+    # junction_is_internal()
+    #
+    # Check whether this loader is specified as internal to
+    # this project.
+    #
+    # Args:
+    #    loader (Loader): The loader to check for
+    #
+    # Returns:
+    #    (bool): Whether the loader is specified as internal
+    #
+    def junction_is_internal(self, loader):
+
+        # Iterate over all paths specified by this project and see
+        # if we find a match for the specified loader.
+        #
+        # Using the regular `Loader.get_loader()` codepath from this
+        # project ensures that we will find the correct loader relative
+        # to this project, regardless of any overrides or link elements
+        # which might have been used in the project.
+        #
+        for internal_path, internal_provenance in self._junction_internal.items():
+            search = self.loader.get_loader(internal_path, internal_provenance, load_subprojects=False)
+            if loader is search:
+                return True
+
+        return False
+
     ########################################################
     #                    Private Methods                   #
     ########################################################
@@ -746,7 +778,9 @@ class Project:
 
         # Junction configuration
         junctions_node = pre_config_node.get_mapping("junctions", default={})
-        junctions_node.validate_keys(["duplicates"])
+        junctions_node.validate_keys(["duplicates", "internal"])
+
+        # Parse duplicates
         junction_duplicates = junctions_node.get_mapping("duplicates", default={})
         for project_name, junctions in junction_duplicates.items():
             # For each junction we preserve the provenance and the junction string,
@@ -756,6 +790,11 @@ class Project:
             for junction_node in junctions:
                 junctions_dict[junction_node.as_str()] = junction_node.get_provenance()
 
+        # Parse internal
+        junction_internal = junctions_node.get_sequence("internal", default=[])
+        for junction_node in junction_internal:
+            self._junction_internal[junction_node.as_str()] = junction_node.get_provenance()
+
         self.loader = Loader(self, parent=parent_loader, provenance=provenance)
 
         self._project_includes = Includes(self.loader, copy_tree=False)