diff options
author | Jonathan Maw <jonathan.maw@codethink.co.uk> | 2019-05-02 17:43:18 +0100 |
---|---|---|
committer | Jonathan Maw <jonathan.maw@codethink.co.uk> | 2019-05-17 17:07:27 +0100 |
commit | dee20725d39d634c804e5e05cd19659e1f945f47 (patch) | |
tree | 020fa0a4ab1700e467712406e7d63cfedce32b0c | |
parent | 8a16aa6f7af238fb77e90c6f4e7d4f1457507b85 (diff) | |
download | buildstream-jonathan/cached-to-artifact.tar.gz |
Delegate storage of cached state to the Artifact classjonathan/cached-to-artifact
This commit also removes Element.__is_cached because it doesn't seem to
serve a purpose.
-rw-r--r-- | buildstream/_artifact.py | 25 | ||||
-rw-r--r-- | buildstream/element.py | 46 |
2 files changed, 40 insertions, 31 deletions
diff --git a/buildstream/_artifact.py b/buildstream/_artifact.py index 2240300c7..fff9b817a 100644 --- a/buildstream/_artifact.py +++ b/buildstream/_artifact.py @@ -66,6 +66,7 @@ class Artifact(): self._metadata_dependencies = None # Dictionary of dependency strong keys from the artifact self._metadata_workspaced = None # Boolean of whether it's a workspaced artifact self._metadata_workspaced_dependencies = None # List of which dependencies are workspaced from the artifact + self.__cached = None # Boolean of whether the artifact is cached # get_files(): # @@ -341,17 +342,20 @@ class Artifact(): # are available, which may depend on command and configuration. The cache # key used for querying is dependant on the current context. # - # This is used by _update_state() to set __strong_cached and __weak_cached. - # # Returns: # (bool): Whether artifact is in local cache # def cached(self): + + if self.__cached is not None: + return self.__cached + context = self._context artifact = self._get_proto() if not artifact: + self.__cached = False return False # Determine whether directories are required @@ -363,8 +367,10 @@ class Artifact(): # Check whether 'files' subdirectory is available, with or without file contents if (require_directories and str(artifact.files) and not self._cas.contains_directory(artifact.files, with_files=require_files)): + self.__cached = False return False + self.__cached = True return True # cached_logs() @@ -387,6 +393,21 @@ class Artifact(): return True + # reset_cached() + # + # Allow the Artifact to query the filesystem to determine whether it + # is cached or not. + # + # NOTE: Due to the fact that a normal buildstream run does not make an + # artifact *not* cached (`bst artifact delete` can do so, but doesn't + # query the Artifact afterwards), it does not update_cached if the + # artifact is already cached. If a cached artifact ever has its key + # changed, this will need to be revisited. + # + def reset_cached(self): + if self.__cached is False: + self.__cached = None + # _get_proto() # # Returns: diff --git a/buildstream/element.py b/buildstream/element.py index 8c507b768..83b40a2db 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -216,8 +216,6 @@ class Element(Plugin): self.__artifacts = context.artifactcache # Artifact cache self.__sourcecache = context.sourcecache # Source cache self.__consistency = Consistency.INCONSISTENT # Cached overall consistency state - self.__strong_cached = None # Whether we have a cached artifact - self.__weak_cached = None # Whether we have a cached artifact self.__assemble_scheduled = False # Element is scheduled to be assembled self.__assemble_done = False # Element is assembled self.__tracking_scheduled = False # Sources are scheduled to be tracked @@ -1055,7 +1053,10 @@ class Element(Plugin): # the artifact cache # def _cached(self): - return self.__is_cached(keystrength=None) + if not self.__artifact: + return False + + return self.__artifact.cached() # _get_build_result(): # @@ -1789,13 +1790,14 @@ class Element(Plugin): # in user context, as to complete a partial artifact pull_buildtrees = self._get_context().pull_buildtrees - if self.__strong_cached and pull_buildtrees: - # If we've specified a subdir, check if the subdir is cached locally - # or if it's possible to get - if self._cached_buildtree() or not self._buildtree_exists(): + if self.__strict_artifact: + if self.__strict_artifact.cached() and pull_buildtrees: + # If we've specified a subdir, check if the subdir is cached locally + # or if it's possible to get + if self._cached_buildtree() or not self._buildtree_exists(): + return False + elif self.__strict_artifact.cached(): return False - elif self.__strong_cached: - return False # Pull is pending if artifact remote server available # and pull has not been attempted yet @@ -2304,18 +2306,12 @@ class Element(Plugin): # have been executed. sandbox._callback(mark_workspace_prepared) - def __is_cached(self, keystrength): - if keystrength is None: - keystrength = _KeyStrength.STRONG if self._get_context().get_strict() else _KeyStrength.WEAK - - return self.__strong_cached if keystrength == _KeyStrength.STRONG else self.__weak_cached - # __assert_cached() # # Raises an error if the artifact is not cached. # def __assert_cached(self, keystrength=None): - assert self.__is_cached(keystrength=keystrength), "{}: Missing artifact {}".format( + assert self._cached(), "{}: Missing artifact {}".format( self, self._get_brief_display_key()) # __get_tainted(): @@ -2882,8 +2878,6 @@ class Element(Plugin): self.__strict_cache_key = None self.__artifact = None self.__strict_artifact = None - self.__weak_cached = None - self.__strong_cached = None # __update_cache_keys() # @@ -2951,8 +2945,6 @@ class Element(Plugin): if not context.get_strict() and not self.__artifact: # We've calculated the weak_key, so instantiate artifact instance member self.__artifact = Artifact(self, context, weak_key=self.__weak_cache_key) - # and update the weak cached state (required early for workspaces) - self.__weak_cached = self.__artifact.cached() if not self.__strict_cache_key: return @@ -2966,15 +2958,11 @@ class Element(Plugin): self.__cache_key = self.__strict_cache_key self.__artifact = self.__strict_artifact - # Query caches now that the weak and strict cache keys are available. - # strong_cached in non-strict mode is only of relevance when querying - # if a 'better' artifact could be pulled, which is redudant if we already - # have it cached locally with a strict_key. As such strong_cached is only - # checked against the 'strict' artifact. - if not self.__strong_cached: - self.__strong_cached = self.__strict_artifact.cached() - if not self.__weak_cached and not context.get_strict(): - self.__weak_cached = self.__artifact.cached() + # Allow caches to be queried, since they may now be cached + # The next invocation of Artifact.cached() will access the filesystem. + # Note that this will safely do nothing if the artifacts are already cached. + self.__strict_artifact.reset_cached() + self.__artifact.reset_cached() # __update_cache_key_non_strict() # |