diff options
author | James Ennis <james.ennis@codethink.co.uk> | 2019-08-22 15:40:35 +0100 |
---|---|---|
committer | James Ennis <james.ennis@codethink.co.uk> | 2019-08-27 15:21:34 +0100 |
commit | bf6ca6615bb5b69defcc41f33625cbfc2b300f1a (patch) | |
tree | ae2d990b5f60997b494a5b18ff3fa82081d2be65 | |
parent | 6c7a2eeec7b648870ffb8384a794208b05661c9a (diff) | |
download | buildstream-bf6ca6615bb5b69defcc41f33625cbfc2b300f1a.tar.gz |
_artifactcache.py: Add remote support to bst artifact show
If remotes exist, each remote will be checked for the target
artifacts. If an artifact is cached remotely, we make a
record of this.
-rw-r--r-- | src/buildstream/_artifactcache.py | 47 | ||||
-rw-r--r-- | src/buildstream/_frontend/widget.py | 2 | ||||
-rw-r--r-- | src/buildstream/_pipeline.py | 12 | ||||
-rw-r--r-- | src/buildstream/_stream.py | 5 | ||||
-rw-r--r-- | src/buildstream/element.py | 11 |
5 files changed, 77 insertions, 0 deletions
diff --git a/src/buildstream/_artifactcache.py b/src/buildstream/_artifactcache.py index f92a7c84f..3357f986a 100644 --- a/src/buildstream/_artifactcache.py +++ b/src/buildstream/_artifactcache.py @@ -383,6 +383,31 @@ class ArtifactCache(BaseCache): return remote_missing_blobs_list + # check_remotes_for_element() + # + # Check if the element is available in any of the remotes + # + # Args: + # element (Element): The element to check + # + # Returns: + # (bool): True if the element is available remotely + # + def check_remotes_for_element(self, element): + # If there are no remotes + if not self._remotes: + return False + + project = element._get_project() + ref = element.get_artifact_name() + for remote in self._remotes[project]: + remote.init() + + if self._query_remote(ref, remote): + return True + + return False + ################################################ # Local Private Methods # ################################################ @@ -520,3 +545,25 @@ class ArtifactCache(BaseCache): f.write(artifact.SerializeToString()) return True + + # _query_remote() + # + # Args: + # ref (str): The artifact ref + # remote (ArtifactRemote): The remote we want to check + # + # Returns: + # (bool): True if the ref exists in the remote, False otherwise. + # + def _query_remote(self, ref, remote): + request = artifact_pb2.GetArtifactRequest() + request.cache_key = ref + try: + artifact_service = artifact_pb2_grpc.ArtifactServiceStub(remote.channel) + artifact_service.GetArtifact(request) + except grpc.RpcError as e: + if e.code() != grpc.StatusCode.NOT_FOUND: + raise ArtifactError("Error when querying: {}".format(e.details())) + return False + + return True diff --git a/src/buildstream/_frontend/widget.py b/src/buildstream/_frontend/widget.py index 85fb00768..e8299868c 100644 --- a/src/buildstream/_frontend/widget.py +++ b/src/buildstream/_frontend/widget.py @@ -856,6 +856,8 @@ class LogLine(Widget): line = p.fmt_subst(line, 'state', "cached", fg='magenta') elif element._cached(): line = p.fmt_subst(line, 'state', "failed", fg='red') + elif element._cached_remotely(): + line = p.fmt_subst(line, 'state', "available", fg='green') else: line = p.fmt_subst(line, 'state', "not cached", fg='bright_red') diff --git a/src/buildstream/_pipeline.py b/src/buildstream/_pipeline.py index 7cf4abbe3..4b0c6ad94 100644 --- a/src/buildstream/_pipeline.py +++ b/src/buildstream/_pipeline.py @@ -154,6 +154,18 @@ class Pipeline(): # dependencies. element._update_ready_for_runtime_and_cached() + # check_remotes() + # + # Check if the target artifact is cached in any of the available remotes + # + # Args: + # targets (list [Element]): The list of element targets + # + def check_remotes(self, targets): + with self._context.messenger.timed_activity("Querying remotes for cached status", silent_nested=True): + for element in targets: + element._cached_remotely() + # dependencies() # # Generator function to iterate over elements and optionally diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index 269cbb542..11f428aaf 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -625,8 +625,13 @@ class Stream(): # Obtain list of Element and/or ArtifactElement objects target_objects = self.load_selection(targets, selection=selection, + use_artifact_config=True, load_refs=True) + if self._artifacts.has_fetch_remotes(): + self._context.disable_fork() + self._pipeline.check_remotes(target_objects) + # XXX: We need to set the name of an ArtifactElement to its ref in order # to display the expected result in the frontend for obj in target_objects: diff --git a/src/buildstream/element.py b/src/buildstream/element.py index c44af942b..10c8320fa 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -219,6 +219,7 @@ class Element(Plugin): self.__updated_strict_cache_keys_of_rdeps = False # Whether we've updated strict cache keys of rdeps self.__ready_for_runtime = False # Whether the element and its runtime dependencies have cache keys self.__ready_for_runtime_and_cached = False # Whether all runtime deps are cached, as well as the element + self.__cached_remotely = None # Whether the element is cached remotely self.__sources = [] # List of Sources self.__weak_cache_key = None # Our cached weak cache key self.__strict_cache_key = None # Our cached cache key for strict builds @@ -1055,6 +1056,16 @@ class Element(Plugin): return self.__artifact.cached() + # _cached_remotely(): + # + # Returns: + # (bool): Whether this element is present in a remote cache + # + def _cached_remotely(self): + if self.__cached_remotely is None: + self.__cached_remotely = self.__artifacts.check_remotes_for_element(self) + return self.__cached_remotely + # _get_build_result(): # # Returns: |