diff options
author | Jürg Billeter <j@bitron.ch> | 2018-01-15 08:52:27 +0100 |
---|---|---|
committer | Tristan Van Berkom <tristan.van.berkom@gmail.com> | 2018-01-16 12:58:39 +0000 |
commit | 03412a9e8a13e6c5f50dec95e067c2ae6900d08f (patch) | |
tree | 2dc7091abd056c761e39c8dc20a294af177c4505 | |
parent | d6ebd6cea97e76e79c2437d4ae93944a983e5727 (diff) | |
download | buildstream-03412a9e8a13e6c5f50dec95e067c2ae6900d08f.tar.gz |
Use explicit source state updatesjuerg/source-state
This adds the _update_state() method to the Source class, similar to the
corresponding method in the Element class.
-rw-r--r-- | buildstream/_pipeline.py | 8 | ||||
-rw-r--r-- | buildstream/_scheduler/fetchqueue.py | 6 | ||||
-rw-r--r-- | buildstream/_scheduler/trackqueue.py | 5 | ||||
-rw-r--r-- | buildstream/element.py | 11 | ||||
-rw-r--r-- | buildstream/source.py | 34 |
5 files changed, 30 insertions, 34 deletions
diff --git a/buildstream/_pipeline.py b/buildstream/_pipeline.py index 52f1d1807..551861e7c 100644 --- a/buildstream/_pipeline.py +++ b/buildstream/_pipeline.py @@ -206,10 +206,10 @@ class Pipeline(): # Load the pipeline in an explicitly inconsistent state, use # this for pipelines with tracking queues enabled. element._schedule_tracking() - else: - # Resolve cache keys and interrogate the artifact cache - # for the first time. - element._update_state() + + # Determine initial element state. This may resolve cache keys + # and interrogate the artifact cache. + element._update_state() # Generator function to iterate over elements and optionally # also iterate over sources. diff --git a/buildstream/_scheduler/fetchqueue.py b/buildstream/_scheduler/fetchqueue.py index 8fc19d283..b284d5b35 100644 --- a/buildstream/_scheduler/fetchqueue.py +++ b/buildstream/_scheduler/fetchqueue.py @@ -57,9 +57,9 @@ class FetchQueue(Queue): if returncode != 0: return False - for source in element.sources(): + element._update_state() - # Successful fetch, we must be CACHED now - source._bump_consistency(Consistency.CACHED) + # Successful fetch, we must be CACHED now + assert(element._consistency() == Consistency.CACHED) return True diff --git a/buildstream/_scheduler/trackqueue.py b/buildstream/_scheduler/trackqueue.py index 3c1521f7b..3f35aff47 100644 --- a/buildstream/_scheduler/trackqueue.py +++ b/buildstream/_scheduler/trackqueue.py @@ -83,13 +83,8 @@ class TrackQueue(Queue): "tracked source to file {}: {}" .format(source, fullname, e)) - # Forcefully recalculate the element's consistency state after successfully - # tracking, this is avoid a following fetch queue operating on the sources - # if the tracked ref is cached as a result. - # context = element._get_context() context._push_message_depth(True) - element._consistency(recalculate=True) element._update_state() context._pop_message_depth() diff --git a/buildstream/element.py b/buildstream/element.py index 54f93091f..c7b13232e 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -717,20 +717,17 @@ class Element(Plugin): # _consistency(): # - # Args: - # recalcualte (bool): Whether to forcefully recalculate - # # Returns: # (list): The minimum consistency of the elements sources # # If the element has no sources, this returns Consistency.CACHED - def _consistency(self, recalculate=False): + def _consistency(self): # The source objects already cache the consistency state, it # should not be expensive to iterate over the sources to get at it consistency = Consistency.CACHED for source in self.__sources: - source_consistency = source._get_consistency(recalculate=recalculate) + source_consistency = source._get_consistency() consistency = min(consistency, source_consistency) return consistency @@ -1374,6 +1371,10 @@ class Element(Plugin): # This must be called whenever the state of an element may have changed. # def _update_state(self): + # Determine consistency of sources + for source in self.__sources: + source._update_state() + if self._consistency() == Consistency.INCONSISTENT: # Tracking is still pending return diff --git a/buildstream/source.py b/buildstream/source.py index 79c1a32b8..d43a57f1c 100644 --- a/buildstream/source.py +++ b/buildstream/source.py @@ -82,7 +82,8 @@ class Source(Plugin): self.__origin_node = meta.origin_node # YAML node this Source was loaded from self.__origin_toplevel = meta.origin_toplevel # Toplevel YAML node for the file self.__origin_filename = meta.origin_filename # Filename of the file the source was loaded from - self.__consistency = None # Cached consistency state + self.__consistency = Consistency.INCONSISTENT # Cached consistency state + self.__tracking = False # Source is scheduled to be tracked self.__workspace = None # Directory of the currently active workspace self.__workspace_key = None # Cached directory content hashes for workspaced source @@ -258,10 +259,15 @@ class Source(Plugin): # Private Methods used in BuildStream # ############################################################# - # Wrapper for get_consistency() api which caches the result + # Update cached consistency for a source # - def _get_consistency(self, recalculate=False): - if recalculate or self.__consistency is None: + # This must be called whenever the state of a source may have changed. + # + def _update_state(self): + if self.__tracking: + return + + if self.__consistency < Consistency.CACHED: self.__consistency = self.get_consistency() if self._has_workspace() and \ @@ -274,6 +280,9 @@ class Source(Plugin): if not os.path.exists(fullpath): self.__consistency = Consistency.INCONSISTENT + # Return cached consistency + # + def _get_consistency(self): return self.__consistency # Return the absolute path of the element's workspace @@ -281,16 +290,7 @@ class Source(Plugin): def _get_workspace_path(self): return os.path.join(self.get_project_directory(), self.__workspace) - # Bump local cached consistency state, this is done from - # the pipeline after the successful completion of fetch - # and track jobs. - # - def _bump_consistency(self, consistency): - if (self.__consistency is None or - consistency > self.__consistency): - self.__consistency = consistency - - # Force a source to appear to be in an inconsistent state. + # Mark a source as scheduled to be tracked # # This is used across the pipeline in sessions where the # source in question are going to be tracked. This is important @@ -299,7 +299,7 @@ class Source(Plugin): # elements from being assembled until the source is CACHED. # def _schedule_tracking(self): - self.__consistency = Consistency.INCONSISTENT + self.__tracking = True # Wrapper function around plugin provided fetch method # @@ -366,6 +366,8 @@ class Source(Plugin): self.set_ref(ref, node) changed = True + self.__tracking = False + return changed # Wrapper for track() @@ -374,8 +376,6 @@ class Source(Plugin): new_ref = self.track() current_ref = self.get_ref() - # It's consistent unless it reported an error - self._bump_consistency(Consistency.RESOLVED) if current_ref != new_ref: self.info("Found new revision: {}".format(new_ref)) |