Stop updating state in queue.status()

Statuses of an element can be changed when:

1) It is pulled
2) It is fetched
3) It is workspaced and it finished building
4) One of its dependencies is tracked
5) One of its dependencies is workspaced and finished building

We can therefore update the statuses at those moments and we don't
need to check all the time.

This reduces considerably the calls to update_states that are done

6 files changed, 214 insertions, 12 deletions

diff --git a/buildstream/_scheduler/queues/buildqueue.py b/buildstream/_scheduler/queues/buildqueue.py
index e63475f05..05e6f7a8b 100644
--- a/buildstream/_scheduler/queues/buildqueue.py
+++ b/buildstream/_scheduler/queues/buildqueue.py
@@ -35,9 +35,6 @@ class BuildQueue(Queue):
         return element._assemble()
 
     def status(self, element):
-        # state of dependencies may have changed, recalculate element state
-        element._update_state()
-
         if not element._is_required():
             # Artifact is not currently required but it may be requested later.
             # Keep it in the queue.
diff --git a/buildstream/_scheduler/queues/fetchqueue.py b/buildstream/_scheduler/queues/fetchqueue.py
index 114790c05..e121f6349 100644
--- a/buildstream/_scheduler/queues/fetchqueue.py
+++ b/buildstream/_scheduler/queues/fetchqueue.py
@@ -44,9 +44,6 @@ class FetchQueue(Queue):
             source._fetch()
 
     def status(self, element):
-        # state of dependencies may have changed, recalculate element state
-        element._update_state()
-
         if not element._is_required():
             # Artifact is not currently required but it may be requested later.
             # Keep it in the queue.
diff --git a/buildstream/_scheduler/queues/pullqueue.py b/buildstream/_scheduler/queues/pullqueue.py
index 2842c5e21..e4868953e 100644
--- a/buildstream/_scheduler/queues/pullqueue.py
+++ b/buildstream/_scheduler/queues/pullqueue.py
@@ -38,9 +38,6 @@ class PullQueue(Queue):
             raise SkipJob(self.action_name)
 
     def status(self, element):
-        # state of dependencies may have changed, recalculate element state
-        element._update_state()
-
         if not element._is_required():
             # Artifact is not currently required but it may be requested later.
             # Keep it in the queue.
diff --git a/buildstream/element.py b/buildstream/element.py
index dd1169f15..9851d042e 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -88,6 +88,7 @@ from ._variables import Variables
 from ._versions import BST_CORE_ARTIFACT_VERSION
 from ._exceptions import BstError, LoadError, LoadErrorReason, ImplError, ErrorDomain
 from .utils import UtilError
+from .types import _UniquePriorityQueue
 from . import Plugin, Consistency
 from . import SandboxFlags
 from . import utils
@@ -214,6 +215,8 @@ class Element(Plugin):
 
         self.__runtime_dependencies = []        # Direct runtime dependency Elements
         self.__build_dependencies = []          # Direct build dependency Elements
+        self.__reverse_dependencies = set()     # Direct reverse dependency Elements
+        self.__ready_for_runtime = False        # Wether the element has all its dependencies ready and has a cache key
         self.__sources = []                     # List of Sources
         self.__weak_cache_key = None            # Our cached weak cache key
         self.__strict_cache_key = None          # Our cached cache key for strict builds
@@ -924,9 +927,12 @@ class Element(Plugin):
         for meta_dep in meta.dependencies:
             dependency = Element._new_from_meta(meta_dep, artifacts)
             element.__runtime_dependencies.append(dependency)
+            dependency.__reverse_dependencies.add(element)
+
         for meta_dep in meta.build_dependencies:
             dependency = Element._new_from_meta(meta_dep, artifacts)
             element.__build_dependencies.append(dependency)
+            dependency.__reverse_dependencies.add(element)
 
         return element
 
@@ -1143,6 +1149,10 @@ class Element(Plugin):
                 # Strong cache key could not be calculated yet
                 return
 
+        if not self.__ready_for_runtime and self.__cache_key is not None:
+            self.__ready_for_runtime = all(
+                dep.__ready_for_runtime for dep in self.__runtime_dependencies)
+
     # _get_display_key():
     #
     # Returns cache keys for display purposes
@@ -1245,7 +1255,7 @@ class Element(Plugin):
         self.__tracking_scheduled = False
         self.__tracking_done = True
 
-        self._update_state()
+        self.__update_state_recursively()
 
     # _track():
     #
@@ -1421,7 +1431,7 @@ class Element(Plugin):
         self.__assemble_scheduled = False
         self.__assemble_done = True
 
-        self._update_state()
+        self.__update_state_recursively()
 
         if self._get_workspace() and self._cached():
             #
@@ -1625,7 +1635,7 @@ class Element(Plugin):
     def _pull_done(self):
         self.__pull_done = True
 
-        self._update_state()
+        self.__update_state_recursively()
 
     def _pull_strong(self, *, progress=None):
         weak_key = self._get_cache_key(strength=_KeyStrength.WEAK)
@@ -2504,6 +2514,24 @@ class Element(Plugin):
 
         return utils._deduplicate(keys)
 
+    # __update_state_recursively()
+    #
+    # Update the state of all reverse dependencies, recursively.
+    #
+    def __update_state_recursively(self):
+        queue = _UniquePriorityQueue()
+        queue.push(self._unique_id, self)
+
+        while queue:
+            element = queue.pop()
+
+            old_ready_for_runtime = element.__ready_for_runtime
+            element._update_state()
+
+            if element.__ready_for_runtime != old_ready_for_runtime:
+                for rdep in element.__reverse_dependencies:
+                    queue.push(rdep._unique_id, rdep)
+
 
 def _overlap_error_detail(f, forbidden_overlap_elements, elements):
     if forbidden_overlap_elements:
diff --git a/buildstream/plugin.py b/buildstream/plugin.py
index f6a10384d..2c94c212c 100644
--- a/buildstream/plugin.py
+++ b/buildstream/plugin.py
@@ -178,6 +178,12 @@ class Plugin():
         # Unique ID
         #
         # This id allows to uniquely identify a plugin.
+        #
+        # /!\ the unique id must be an increasing value /!\
+        # This is because we are depending on it in buildstream.element.Element
+        # to give us a topological sort over all elements.
+        # Modifying how we handle ids here will modify the behavior of the
+        # Element's state handling.
         self._unique_id = next(self.__id_generator)
 
         # register ourself in the table containing all existing plugins
diff --git a/buildstream/types.py b/buildstream/types.py
new file mode 100644
index 000000000..d54bf0b6e
--- /dev/null
+++ b/buildstream/types.py
@@ -0,0 +1,177 @@
+#
+#  Copyright (C) 2018 Bloomberg LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+#        Jim MacArthur <jim.macarthur@codethink.co.uk>
+#        Benjamin Schubert <bschubert15@bloomberg.net>
+
+"""
+Foundation types
+================
+
+"""
+
+from enum import Enum
+import heapq
+
+
+class Scope(Enum):
+    """Defines the scope of dependencies to include for a given element
+    when iterating over the dependency graph in APIs like
+    :func:`Element.dependencies() <buildstream.element.Element.dependencies>`
+    """
+
+    ALL = 1
+    """All elements which the given element depends on, following
+    all elements required for building. Including the element itself.
+    """
+
+    BUILD = 2
+    """All elements required for building the element, including their
+    respective run dependencies. Not including the given element itself.
+    """
+
+    RUN = 3
+    """All elements required for running the element. Including the element
+    itself.
+    """
+
+    NONE = 4
+    """Just the element itself, no dependencies.
+
+    *Since: 1.4*
+    """
+
+
+class Consistency():
+    """Defines the various consistency states of a :class:`.Source`.
+    """
+
+    INCONSISTENT = 0
+    """Inconsistent
+
+    Inconsistent sources have no explicit reference set. They cannot
+    produce a cache key, be fetched or staged. They can only be tracked.
+    """
+
+    RESOLVED = 1
+    """Resolved
+
+    Resolved sources have a reference and can produce a cache key and
+    be fetched, however they cannot be staged.
+    """
+
+    CACHED = 2
+    """Cached
+
+    Sources have a cached unstaged copy in the source directory.
+    """
+
+
+class CoreWarnings():
+    """CoreWarnings()
+
+    Some common warnings which are raised by core functionalities within BuildStream are found in this class.
+    """
+
+    OVERLAPS = "overlaps"
+    """
+    This warning will be produced when buildstream detects an overlap on an element
+        which is not whitelisted. See :ref:`Overlap Whitelist <public_overlap_whitelist>`
+    """
+
+    REF_NOT_IN_TRACK = "ref-not-in-track"
+    """
+    This warning will be produced when a source is configured with a reference
+    which is found to be invalid based on the configured track
+    """
+
+    BAD_ELEMENT_SUFFIX = "bad-element-suffix"
+    """
+    This warning will be produced when an element whose name does not end in .bst
+    is referenced either on the command line or by another element
+    """
+
+    BAD_CHARACTERS_IN_NAME = "bad-characters-in-name"
+    """
+    This warning will be produces when filename for a target contains invalid
+    characters in its name.
+    """
+
+
+# _KeyStrength():
+#
+# Strength of cache key
+#
+class _KeyStrength(Enum):
+
+    # Includes strong cache keys of all build dependencies and their
+    # runtime dependencies.
+    STRONG = 1
+
+    # Includes names of direct build dependencies but does not include
+    # cache keys of dependencies.
+    WEAK = 2
+
+
+# _UniquePriorityQueue():
+#
+# Implements a priority queue that adds only each key once.
+#
+# The queue will store and priority based on a tuple (key, item).
+#
+class _UniquePriorityQueue:
+
+    def __init__(self):
+        self._items = set()
+        self._heap = []
+
+    # push():
+    #
+    # Push a new item in the queue.
+    #
+    # If the item is already present in the queue as identified by the key,
+    # this is a noop.
+    #
+    # Args:
+    #     key (hashable, comparable): unique key to use for checking for
+    #                                 the object's existence and used for
+    #                                 ordering
+    #     item (any): item to push to the queue
+    #
+    def push(self, key, item):
+        if key not in self._items:
+            self._items.add(key)
+            heapq.heappush(self._heap, (key, item))
+
+    # pop():
+    #
+    # Pop the next item from the queue, by priority order.
+    #
+    # Returns:
+    #     (any): the next item
+    #
+    # Throw:
+    #     IndexError: when the list is empty
+    #
+    def pop(self):
+        key, item = heapq.heappop(self._heap)
+        self._items.remove(key)
+        return item
+
+    def __len__(self):
+        return len(self._heap)