diff options
author | bst-marge-bot <marge-bot@buildstream.build> | 2019-06-24 16:47:33 +0000 |
---|---|---|
committer | bst-marge-bot <marge-bot@buildstream.build> | 2019-06-24 16:47:33 +0000 |
commit | 722d456799221ab57742b6e199a79d8476c0af98 (patch) | |
tree | 9797c6957818d98934e49e801348684280e6d3f4 | |
parent | 5c8642b4c86fed13b0b29aaa01e21871d3a89a31 (diff) | |
parent | 14efa5c06ee380fa524091b8e6eb18b3713c332f (diff) | |
download | buildstream-722d456799221ab57742b6e199a79d8476c0af98.tar.gz |
Merge branch 'jennis/notify_reverse_deps' into 'master'
Notify direct reverse dependencies when an Element becomes ready for runtime and cached
See merge request BuildStream/buildstream!1406
-rw-r--r-- | src/buildstream/_pipeline.py | 5 | ||||
-rw-r--r-- | src/buildstream/element.py | 112 |
2 files changed, 93 insertions, 24 deletions
diff --git a/src/buildstream/_pipeline.py b/src/buildstream/_pipeline.py index d44813348..e6ae94cfd 100644 --- a/src/buildstream/_pipeline.py +++ b/src/buildstream/_pipeline.py @@ -138,6 +138,11 @@ class Pipeline(): # Determine initial element state. element._update_state() + # We may already have Elements which are cached and have their runtimes + # cached, if this is the case, we should immediately notify their reverse + # dependencies. + element._update_ready_for_runtime_and_cached() + # dependencies() # # Generator function to iterate over elements and optionally diff --git a/src/buildstream/element.py b/src/buildstream/element.py index a60546084..60aefecad 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -206,9 +206,12 @@ class Element(Plugin): self.__runtime_dependencies = [] # Direct runtime dependency Elements self.__build_dependencies = [] # Direct build dependency Elements - self.__reverse_dependencies = set() # Direct reverse dependency Elements + self.__reverse_build_deps = set() # Direct reverse build dependency Elements + self.__reverse_runtime_deps = set() # Direct reverse runtime dependency Elements + self.__remaining_build_deps_uncached = None # Built dependencies which are not yet cached + self.__remaining_runtime_deps_uncached = None # Runtime dependencies which are not yet cached self.__ready_for_runtime = False # Whether the element has all dependencies ready and has a cache key - self.__ready_for_runtime_and_cached = False # Whether the element has all deps ready for runtime and cached + self.__ready_for_runtime_and_cached = False # Whether all runtime deps are cached, as well as the element self.__sources = [] # List of Sources self.__weak_cache_key = None # Our cached weak cache key self.__strict_cache_key = None # Our cached cache key for strict builds @@ -1003,12 +1006,14 @@ class Element(Plugin): for meta_dep in meta.dependencies: dependency = Element._new_from_meta(meta_dep) element.__runtime_dependencies.append(dependency) - dependency.__reverse_dependencies.add(element) + dependency.__reverse_runtime_deps.add(element) + element.__remaining_runtime_deps_uncached = len(element.__runtime_dependencies) for meta_dep in meta.build_dependencies: dependency = Element._new_from_meta(meta_dep) element.__build_dependencies.append(dependency) - dependency.__reverse_dependencies.add(element) + dependency.__reverse_build_deps.add(element) + element.__remaining_build_deps_uncached = len(element.__build_dependencies) return element @@ -1131,7 +1136,7 @@ class Element(Plugin): if not self.__assemble_scheduled: return False - return all(dep.__ready_for_runtime_and_cached for dep in self.dependencies(Scope.BUILD, recurse=False)) + return self.__remaining_build_deps_uncached == 0 # _get_cache_key(): # @@ -1193,10 +1198,6 @@ class Element(Plugin): # The correct keys can only be calculated once the build is complete self.__reset_cache_data() - if self.__buildable_callback is not None and self._buildable(): - self.__buildable_callback(self) - self.__buildable_callback = None - return self.__update_cache_keys() @@ -1220,6 +1221,7 @@ class Element(Plugin): # key is not available yet. if self.__can_query_cache_callback is not None: self.__can_query_cache_callback(self) + self.__can_query_cache_callback = None return @@ -1230,17 +1232,6 @@ class Element(Plugin): self.__ready_for_runtime = all( dep.__ready_for_runtime for dep in self.__runtime_dependencies) - if self.__ready_for_runtime: - # ready_for_runtime_and_cached is stronger than ready_for_runtime, so don't - # check the former if the latter is False - if not self.__ready_for_runtime_and_cached and self._cached_success(): - self.__ready_for_runtime_and_cached = all( - dep.__ready_for_runtime_and_cached for dep in self.__runtime_dependencies) - - if self.__buildable_callback is not None and self._buildable(): - self.__buildable_callback(self) - self.__buildable_callback = None - # _get_display_key(): # # Returns cache keys for display purposes @@ -1600,6 +1591,7 @@ class Element(Plugin): self.__artifact.reset_cached() self.__update_state_recursively() + self._update_ready_for_runtime_and_cached() if self._get_workspace() and self._cached_success(): assert utils._is_main_process(), \ @@ -1856,6 +1848,7 @@ class Element(Plugin): self.__artifact.reset_cached() self.__update_state_recursively() + self._update_ready_for_runtime_and_cached() # _pull(): # @@ -2322,6 +2315,32 @@ class Element(Plugin): def _set_depth(self, depth): self._depth = depth + # _update_ready_for_runtime_and_cached() + # + # An Element becomes ready for runtime and cached once the following three criteria + # are met: + # 1. The Element has a strong cache key + # 2. The Element is cached (locally) + # 3. The runtime dependencies of the Element are ready for runtime and cached. + # + # These three criteria serve as potential trigger points as to when an Element may have + # become ready for runtime and cached. + # + # Once an Element becomes ready for runtime and cached, we notify the reverse + # runtime dependencies and the reverse build dependencies of the element, decrementing + # the appropriate counters. + # + def _update_ready_for_runtime_and_cached(self): + if not self.__ready_for_runtime_and_cached: + if self.__remaining_runtime_deps_uncached == 0 and self.__cache_key and self._cached_success(): + self.__ready_for_runtime_and_cached = True + + # Notify reverse dependencies + for rdep in self.__reverse_runtime_deps: + rdep.__on_runtime_dependency_ready_for_runtime_and_cached() + for rdep in self.__reverse_build_deps: + rdep.__on_build_dependency_ready_for_runtime_and_cached() + ############################################################# # Private Local Methods # ############################################################# @@ -2960,16 +2979,46 @@ class Element(Plugin): element = queue.pop() old_ready_for_runtime = element.__ready_for_runtime - old_ready_for_runtime_and_cached = element.__ready_for_runtime_and_cached old_strict_cache_key = element.__strict_cache_key element._update_state() if element.__ready_for_runtime != old_ready_for_runtime or \ - element.__ready_for_runtime_and_cached != old_ready_for_runtime_and_cached or \ element.__strict_cache_key != old_strict_cache_key: - for rdep in element.__reverse_dependencies: + for rdep in element.__reverse_build_deps | element.__reverse_runtime_deps: queue.push(rdep._unique_id, rdep) + # __on_runtime_dependency_ready_for_runtime_and_cached() + # + # This function is called once one of the Element's runtime dependencies has + # become ready for runtime and cached. + # + # On calling this function, we decrement the Element's remaining runtime deps counter. + # If this is zero, we attempt to notify all reverse dependencies of the Element. + # + def __on_runtime_dependency_ready_for_runtime_and_cached(self): + self.__remaining_runtime_deps_uncached -= 1 + assert not self.__remaining_runtime_deps_uncached < 0 + + # Try to notify reverse dependencies if all runtime deps are ready + if self.__remaining_runtime_deps_uncached == 0: + self._update_ready_for_runtime_and_cached() + + # __on_build_dependency_ready_for_runtime_and_cached() + # + # This function is called once one of the Element's build dependencies has become + # ready for runtime and cached. + # + # On calling this function, we decrement the Element's remaining build deps counter. + # If this is zero, we invoke the buildable callback. + # + def __on_build_dependency_ready_for_runtime_and_cached(self): + self.__remaining_build_deps_uncached -= 1 + assert not self.__remaining_build_deps_uncached < 0 + + if self.__buildable_callback is not None and self._buildable(): + self.__buildable_callback(self) + self.__buildable_callback = None + # __reset_cache_data() # # Resets all data related to cache key calculation and whether an artifact @@ -3006,6 +3055,8 @@ class Element(Plugin): # has changed. # def __update_cache_keys(self): + context = self._get_context() + if self.__weak_cache_key is None: # Calculate weak cache key # Weak cache key includes names of direct build dependencies @@ -3033,6 +3084,15 @@ class Element(Plugin): ] self.__strict_cache_key = self._calculate_cache_key(dependencies) + # In strict mode, the strong cache key always matches the strict cache key + if context.get_strict(): + self.__cache_key = self.__strict_cache_key + + # If the element is cached, and has all of its runtime dependencies cached, + # now that we have the cache key, we are able to notify reverse dependencies + # that the element it ready. This is a likely trigger for workspaced elements. + self._update_ready_for_runtime_and_cached() + if self.__strict_cache_key is not None and self.__can_query_cache_callback is not None: self.__can_query_cache_callback(self) self.__can_query_cache_callback = None @@ -3068,7 +3128,6 @@ class Element(Plugin): # In strict mode, the strong cache key always matches the strict cache key if context.get_strict(): - self.__cache_key = self.__strict_cache_key self.__artifact = self.__strict_artifact # __update_cache_key_non_strict() @@ -3107,6 +3166,11 @@ class Element(Plugin): # Strong cache key could not be calculated yet return + # If the element is cached, and has all of its runtime dependencies cached, + # now that we have the strong cache key, we are able to notify reverse dependencies + # that the element it ready. This is a likely trigger for workspaced elements. + self._update_ready_for_runtime_and_cached() + # Now we have the strong cache key, update the Artifact self.__artifact._cache_key = self.__cache_key |