From bf6ca6615bb5b69defcc41f33625cbfc2b300f1a Mon Sep 17 00:00:00 2001 From: James Ennis Date: Thu, 22 Aug 2019 15:40:35 +0100 Subject: _artifactcache.py: Add remote support to bst artifact show If remotes exist, each remote will be checked for the target artifacts. If an artifact is cached remotely, we make a record of this. --- src/buildstream/_artifactcache.py | 47 +++++++++++++++++++++++++++++++++++++ src/buildstream/_frontend/widget.py | 2 ++ src/buildstream/_pipeline.py | 12 ++++++++++ src/buildstream/_stream.py | 5 ++++ src/buildstream/element.py | 11 +++++++++ 5 files changed, 77 insertions(+) diff --git a/src/buildstream/_artifactcache.py b/src/buildstream/_artifactcache.py index f92a7c84f..3357f986a 100644 --- a/src/buildstream/_artifactcache.py +++ b/src/buildstream/_artifactcache.py @@ -383,6 +383,31 @@ class ArtifactCache(BaseCache): return remote_missing_blobs_list + # check_remotes_for_element() + # + # Check if the element is available in any of the remotes + # + # Args: + # element (Element): The element to check + # + # Returns: + # (bool): True if the element is available remotely + # + def check_remotes_for_element(self, element): + # If there are no remotes + if not self._remotes: + return False + + project = element._get_project() + ref = element.get_artifact_name() + for remote in self._remotes[project]: + remote.init() + + if self._query_remote(ref, remote): + return True + + return False + ################################################ # Local Private Methods # ################################################ @@ -520,3 +545,25 @@ class ArtifactCache(BaseCache): f.write(artifact.SerializeToString()) return True + + # _query_remote() + # + # Args: + # ref (str): The artifact ref + # remote (ArtifactRemote): The remote we want to check + # + # Returns: + # (bool): True if the ref exists in the remote, False otherwise. + # + def _query_remote(self, ref, remote): + request = artifact_pb2.GetArtifactRequest() + request.cache_key = ref + try: + artifact_service = artifact_pb2_grpc.ArtifactServiceStub(remote.channel) + artifact_service.GetArtifact(request) + except grpc.RpcError as e: + if e.code() != grpc.StatusCode.NOT_FOUND: + raise ArtifactError("Error when querying: {}".format(e.details())) + return False + + return True diff --git a/src/buildstream/_frontend/widget.py b/src/buildstream/_frontend/widget.py index 85fb00768..e8299868c 100644 --- a/src/buildstream/_frontend/widget.py +++ b/src/buildstream/_frontend/widget.py @@ -856,6 +856,8 @@ class LogLine(Widget): line = p.fmt_subst(line, 'state', "cached", fg='magenta') elif element._cached(): line = p.fmt_subst(line, 'state', "failed", fg='red') + elif element._cached_remotely(): + line = p.fmt_subst(line, 'state', "available", fg='green') else: line = p.fmt_subst(line, 'state', "not cached", fg='bright_red') diff --git a/src/buildstream/_pipeline.py b/src/buildstream/_pipeline.py index 7cf4abbe3..4b0c6ad94 100644 --- a/src/buildstream/_pipeline.py +++ b/src/buildstream/_pipeline.py @@ -154,6 +154,18 @@ class Pipeline(): # dependencies. element._update_ready_for_runtime_and_cached() + # check_remotes() + # + # Check if the target artifact is cached in any of the available remotes + # + # Args: + # targets (list [Element]): The list of element targets + # + def check_remotes(self, targets): + with self._context.messenger.timed_activity("Querying remotes for cached status", silent_nested=True): + for element in targets: + element._cached_remotely() + # dependencies() # # Generator function to iterate over elements and optionally diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index 269cbb542..11f428aaf 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -625,8 +625,13 @@ class Stream(): # Obtain list of Element and/or ArtifactElement objects target_objects = self.load_selection(targets, selection=selection, + use_artifact_config=True, load_refs=True) + if self._artifacts.has_fetch_remotes(): + self._context.disable_fork() + self._pipeline.check_remotes(target_objects) + # XXX: We need to set the name of an ArtifactElement to its ref in order # to display the expected result in the frontend for obj in target_objects: diff --git a/src/buildstream/element.py b/src/buildstream/element.py index c44af942b..10c8320fa 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -219,6 +219,7 @@ class Element(Plugin): self.__updated_strict_cache_keys_of_rdeps = False # Whether we've updated strict cache keys of rdeps self.__ready_for_runtime = False # Whether the element and its runtime dependencies have cache keys self.__ready_for_runtime_and_cached = False # Whether all runtime deps are cached, as well as the element + self.__cached_remotely = None # Whether the element is cached remotely self.__sources = [] # List of Sources self.__weak_cache_key = None # Our cached weak cache key self.__strict_cache_key = None # Our cached cache key for strict builds @@ -1055,6 +1056,16 @@ class Element(Plugin): return self.__artifact.cached() + # _cached_remotely(): + # + # Returns: + # (bool): Whether this element is present in a remote cache + # + def _cached_remotely(self): + if self.__cached_remotely is None: + self.__cached_remotely = self.__artifacts.check_remotes_for_element(self) + return self.__cached_remotely + # _get_build_result(): # # Returns: -- cgit v1.2.1