element.py: Cache source consistency logic and refactor workspace logic into Element

Workspaces are now element wide, so consistency edge cases must
be handled at the element level instead of the source level.

6 files changed, 55 insertions, 47 deletions

diff --git a/buildstream/_frontend/app.py b/buildstream/_frontend/app.py
index d5c98564a..14a8201d9 100644
--- a/buildstream/_frontend/app.py
+++ b/buildstream/_frontend/app.py
@@ -416,7 +416,7 @@ class App():
         if not no_checkout or track_first:
             self.pipeline.fetch(self.scheduler, [target], track_first)
 
-        if not no_checkout and target._consistency() != Consistency.CACHED:
+        if not no_checkout and target._get_consistency() != Consistency.CACHED:
             raise PipelineError("Could not stage uncached source. " +
                                 "Use `--track` to track and " +
                                 "fetch the latest version of the " +
diff --git a/buildstream/_frontend/widget.py b/buildstream/_frontend/widget.py
index 52eff0ab8..53509b31c 100644
--- a/buildstream/_frontend/widget.py
+++ b/buildstream/_frontend/widget.py
@@ -655,7 +655,7 @@ class LogLine(Widget):
             line = p.fmt_subst(line, 'key', cache_key, fg='yellow', dim=dim_keys)
             line = p.fmt_subst(line, 'full-key', full_key, fg='yellow', dim=dim_keys)
 
-            consistency = element._consistency()
+            consistency = element._get_consistency()
             if consistency == Consistency.INCONSISTENT:
                 line = p.fmt_subst(line, 'state', "no reference", fg='red')
             else:
diff --git a/buildstream/_pipeline.py b/buildstream/_pipeline.py
index a8de2c2d6..98ea6fbe8 100644
--- a/buildstream/_pipeline.py
+++ b/buildstream/_pipeline.py
@@ -263,7 +263,7 @@ class Pipeline():
             self._assert_consistent(plan)
 
         # Filter out elements with cached sources, we already have them.
-        cached = [elt for elt in plan if elt._consistency() == Consistency.CACHED]
+        cached = [elt for elt in plan if elt._get_consistency() == Consistency.CACHED]
         plan = [elt for elt in plan if elt not in cached]
 
         self.session_elements = len(plan)
@@ -718,7 +718,7 @@ class Pipeline():
         inconsistent = []
         with self.context.timed_activity("Checking sources"):
             for element in toplevel:
-                if element._consistency() == Consistency.INCONSISTENT:
+                if element._get_consistency() == Consistency.INCONSISTENT:
                     inconsistent.append(element)
 
         if inconsistent:
diff --git a/buildstream/_scheduler/fetchqueue.py b/buildstream/_scheduler/fetchqueue.py
index bdd8e14f5..159d122aa 100644
--- a/buildstream/_scheduler/fetchqueue.py
+++ b/buildstream/_scheduler/fetchqueue.py
@@ -57,7 +57,7 @@ class FetchQueue(Queue):
 
         # This will automatically skip elements which
         # have no sources.
-        if element._consistency() == Consistency.CACHED:
+        if element._get_consistency() == Consistency.CACHED:
             return QueueStatus.SKIP
 
         return QueueStatus.READY
@@ -70,6 +70,6 @@ class FetchQueue(Queue):
         element._update_state()
 
         # Successful fetch, we must be CACHED now
-        assert element._consistency() == Consistency.CACHED
+        assert element._get_consistency() == Consistency.CACHED
 
         return True
diff --git a/buildstream/element.py b/buildstream/element.py
index 76a6e0b73..c4a534065 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -149,6 +149,7 @@ class Element(Plugin):
         self.__weak_cache_key = None            # Our cached weak cache key
         self.__strict_cache_key = None          # Our cached cache key for strict builds
         self.__artifacts = artifacts            # Artifact cache
+        self.__consistency = Consistency.INCONSISTENT  # Cached overall consistency state
         self.__cached = None                    # Whether we have a cached artifact
         self.__strong_cached = None             # Whether we have a cached artifact
         self.__remotely_cached = None           # Whether we have a remotely cached artifact
@@ -760,15 +761,6 @@ class Element(Plugin):
     #            Private Methods used in BuildStream            #
     #############################################################
 
-    # _get_workspace():
-    #
-    # Returns:
-    #    (Workspace|None): A workspace associated with this element
-    #
-    def _get_workspace(self):
-        project = self._get_project()
-        return project.workspaces.get_workspace(self.name)
-
     # _get_artifact_metadata():
     #
     # Retrieve metadata from the given artifact.
@@ -826,22 +818,6 @@ class Element(Plugin):
         if scope != Scope.BUILD:
             self.__runtime_dependencies.append(dependency)
 
-    # _consistency():
-    #
-    # Returns:
-    #    (list): The minimum consistency of the elements sources
-    #
-    # If the element has no sources, this returns Consistency.CACHED
-    def _consistency(self):
-
-        # The source objects already cache the consistency state, it
-        # should not be expensive to iterate over the sources to get at it
-        consistency = Consistency.CACHED
-        for source in self.__sources:
-            source_consistency = source._get_consistency()
-            consistency = min(consistency, source_consistency)
-        return consistency
-
     # _schedule_tracking():
     #
     # Force an element state to be inconsistent. Any sources appear to be
@@ -963,7 +939,7 @@ class Element(Plugin):
     #    (bool): Whether this element can currently be built
     #
     def _buildable(self):
-        if self._consistency() != Consistency.CACHED:
+        if self._get_consistency() != Consistency.CACHED:
             return False
 
         for dependency in self.dependencies(Scope.BUILD):
@@ -1351,10 +1327,21 @@ class Element(Plugin):
         for source in self.sources():
             source._set_workspace(path)
 
+    # _get_workspace():
+    #
+    # Returns:
+    #    (Workspace|None): A workspace associated with this element
+    #
+    def _get_workspace(self):
+        project = self._get_project()
+        return project.workspaces.get_workspace(self.name)
+
     # Whether this element has a source that is workspaced.
     #
     def _workspaced(self):
-        return any(source._has_workspace() for source in self.sources())
+        if self._get_workspace():
+            return True
+        return False
 
     # _workspaced_artifact():
     #
@@ -1618,6 +1605,38 @@ class Element(Plugin):
     def _pull_failed(self):
         self.__pull_failed = True
 
+    # _get_consistency()
+    #
+    # Returns cached consistency state
+    #
+    def _get_consistency(self):
+        return self.__consistency
+
+    # __update_source_state()
+    #
+    # Updates source consistency state
+    #
+    def __update_source_state(self):
+
+        # Determine overall consistency of the element
+        consistency = Consistency.CACHED
+        for source in self.__sources:
+            source._update_state()
+            source_consistency = source._get_consistency()
+            consistency = min(consistency, source_consistency)
+        self.__consistency = consistency
+
+        # Special case for workspaces
+        workspace = self._get_workspace()
+        if workspace and self.__consistency > Consistency.INCONSISTENT:
+
+            # A workspace is considered inconsistent in the case
+            # that it's directory went missing
+            #
+            fullpath = workspace.get_absolute_path()
+            if not os.path.exists(fullpath):
+                self.__consistency = Consistency.INCONSISTENT
+
     # _update_state()
     #
     # Keep track of element state. Calculate cache keys if possible and
@@ -1628,11 +1647,10 @@ class Element(Plugin):
     def _update_state(self):
         context = self._get_context()
 
-        # Determine consistency of sources
-        for source in self.__sources:
-            source._update_state()
+        # Compute and determine consistency of sources
+        self.__update_source_state()
 
-        if self._consistency() == Consistency.INCONSISTENT:
+        if self._get_consistency() == Consistency.INCONSISTENT:
             # Tracking is still pending
             return
 
diff --git a/buildstream/source.py b/buildstream/source.py
index b9cb23e83..9d5449fbc 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -321,16 +321,6 @@ class Source(Plugin):
             with context.silence():
                 self.__consistency = self.get_consistency()
 
-            if self._has_workspace() and \
-               self.__consistency > Consistency.INCONSISTENT:
-
-                # A workspace is considered inconsistent in the case
-                # that it's directory went missing
-                #
-                fullpath = self.__workspace.get_absolute_path()
-                if not os.path.exists(fullpath):
-                    self.__consistency = Consistency.INCONSISTENT
-
     # Return cached consistency
     #
     def _get_consistency(self):