element.py, _pipeline.py: Moved instantiation codepath to Element class methods.

This will allow the instantiation codepath to be shared by the Loader
which also needs to instantiate elements for junctions.

2 files changed, 74 insertions, 54 deletions

diff --git a/buildstream/_pipeline.py b/buildstream/_pipeline.py
index 6fd05db87..e99b4a540 100644
--- a/buildstream/_pipeline.py
+++ b/buildstream/_pipeline.py
@@ -30,6 +30,7 @@ from tempfile import TemporaryDirectory
 from ._exceptions import PipelineError, ImplError, BstError
 from ._message import Message, MessageType
 from ._loader import Loader
+from .element import Element
 from . import Consistency
 from . import Scope
 from . import _site
@@ -80,8 +81,6 @@ class Pipeline():
         #
         # Private members
         #
-        self._resolved_elements = {}
-        self._redundant_refs = []
         self._artifacts = None
         self._loader = None
         self._exceptions = None
@@ -99,18 +98,21 @@ class Pipeline():
         with self.context.timed_activity("Loading pipeline", silent_nested=True):
             meta_elements = self._loader.load(rewritable, None)
 
-        # Resolve the real elements now that we've resolved the project
+        # Resolve the real elements now that we've loaded the project
         with self.context.timed_activity("Resolving pipeline"):
-            resolved_elements = [self._resolve(meta_element)
-                                 for meta_element in meta_elements]
+            resolved_elements = [
+                Element._new_from_meta(meta, self._artifacts)
+                for meta in meta_elements
+            ]
 
         # Now warn about any redundant source references which may have
         # been discovered in the resolve() phase.
-        if self._redundant_refs:
+        redundant_refs = Element._get_redundant_source_refs()
+        if redundant_refs:
             detail = "The following inline specified source references will be ignored:\n\n"
             lines = [
                 "{}:{}".format(source._get_provenance(), ref)
-                for source, ref in self._redundant_refs
+                for source, ref in redundant_refs
             ]
             detail += "\n".join(lines)
             self._message(MessageType.WARN, "Ignoring redundant source references", detail=detail)
@@ -171,6 +173,9 @@ class Pipeline():
         if self._loader:
             self._loader.cleanup()
 
+        # Reset the element loader state
+        Element._reset_load_state()
+
     # deps_elements()
     #
     # Args:
@@ -621,40 +626,6 @@ class Pipeline():
 
         return track_elements
 
-    # _resolve()
-    #
-    # Instantiates plugin-provided Element and Source instances
-    # from MetaElement and MetaSource objects
-    #
-    # This has a side effect of populating `self._redundant_refs` so
-    # we can later print a warning
-    #
-    def _resolve(self, meta_element):
-        if meta_element in self._resolved_elements:
-            return self._resolved_elements[meta_element]
-
-        element = meta_element.project.create_element(self._artifacts, meta_element)
-
-        self._resolved_elements[meta_element] = element
-
-        # resolve dependencies
-        for dep in meta_element.dependencies:
-            element._add_dependency(self._resolve(dep), Scope.RUN)
-        for dep in meta_element.build_dependencies:
-            element._add_dependency(self._resolve(dep), Scope.BUILD)
-
-        # resolve sources
-        for meta_source in meta_element.sources:
-            source = meta_element.project.create_source(meta_source)
-            redundant_ref = source._load_ref()
-            element._add_source(source)
-
-            # Collect redundant refs for a warning message
-            if redundant_ref is not None:
-                self._redundant_refs.append((source, redundant_ref))
-
-        return element
-
     # _prefilght()
     #
     # Preflights all the plugins in the pipeline
diff --git a/buildstream/element.py b/buildstream/element.py
index f878c50dc..324efa86c 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -154,8 +154,10 @@ class Element(Plugin):
     All elements derive from this class, this interface defines how
     the core will be interacting with Elements.
     """
-    __defaults = {}          # The defaults from the yaml file and project
-    __defaults_set = False   # Flag, in case there are no defaults at all
+    __defaults = {}               # The defaults from the yaml file and project
+    __defaults_set = False        # Flag, in case there are no defaults at all
+    __instantiated_elements = {}  # A hash of Element by MetaElement
+    __redundant_source_refs = []  # A list of (source, ref) tuples which were redundantly specified
 
     BST_ARTIFACT_VERSION = 0
     """The element plugin's artifact version
@@ -851,22 +853,69 @@ class Element(Plugin):
     #############################################################
     #            Private Methods used in BuildStream            #
     #############################################################
-    # _add_source():
+
+    # _new_from_meta():
+    #
+    # Recursively instantiate a new Element instance, it's sources
+    # and it's dependencies from a meta element.
+    #
+    # Args:
+    #    artifacts (ArtifactCache): The artifact cache
+    #    meta (MetaElement): The meta element
+    #
+    # Returns:
+    #    (Element): A newly created Element instance
+    #
+    @classmethod
+    def _new_from_meta(cls, meta, artifacts):
+
+        if meta in cls.__instantiated_elements:
+            return cls.__instantiated_elements[meta]
+
+        project = meta.project
+        element = project.create_element(artifacts, meta)
+        cls.__instantiated_elements[meta] = element
+
+        # Instantiate sources
+        for meta_source in meta.sources:
+            source = project.create_source(meta_source)
+            redundant_ref = source._load_ref()
+            element.__sources.append(source)
+
+            # Collect redundant refs which occurred at load time
+            if redundant_ref is not None:
+                cls.__redundant_source_refs.append((source, redundant_ref))
+
+        # Instantiate dependencies
+        for meta_dep in meta.dependencies:
+            dependency = Element._new_from_meta(meta_dep, artifacts)
+            element.__runtime_dependencies.append(dependency)
+        for meta_dep in meta.build_dependencies:
+            dependency = Element._new_from_meta(meta_dep, artifacts)
+            element.__build_dependencies.append(dependency)
+
+        return element
+
+    # _get_redundant_source_refs()
     #
-    # Adds a source, for pipeline construction
+    # Fetches a list of (Source, ref) tuples of all the Sources
+    # which were loaded with a ref specified in the element declaration
+    # for projects which use project.refs ref-storage.
     #
-    def _add_source(self, source):
-        self.__sources.append(source)
+    # This is used to produce a warning
+    @classmethod
+    def _get_redundant_source_refs(cls):
+        return cls.__redundant_source_refs
 
-    # _add_dependency()
+    # _reset_load_state()
     #
-    # Adds a dependency, for pipeline construction
+    # This is called by Pipeline.cleanup() and is used to
+    # reset the loader state between multiple sessions.
     #
-    def _add_dependency(self, dependency, scope):
-        if scope != Scope.RUN:
-            self.__build_dependencies.append(dependency)
-        if scope != Scope.BUILD:
-            self.__runtime_dependencies.append(dependency)
+    @classmethod
+    def _reset_load_state(cls):
+        cls.__instantiated_elements = {}
+        cls.__redundant_source_refs = []
 
     # _get_consistency()
     #