diff options
Diffstat (limited to 'src/buildstream/_artifactcache.py')
| -rw-r--r-- | src/buildstream/_artifactcache.py | 79 |
1 files changed, 19 insertions, 60 deletions
diff --git a/src/buildstream/_artifactcache.py b/src/buildstream/_artifactcache.py index 2eb738db1..40b23e126 100644 --- a/src/buildstream/_artifactcache.py +++ b/src/buildstream/_artifactcache.py @@ -195,12 +195,7 @@ class ArtifactCache(BaseCache): # ([str]) - A list of artifact names as generated in LRU order # def list_artifacts(self, *, glob=None): - return [ - ref - for _, ref in sorted( - list(self._list_refs_mtimes(self.artifactdir, glob_expr=glob)) - ) - ] + return [ref for _, ref in sorted(list(self._list_refs_mtimes(self.artifactdir, glob_expr=glob)))] # remove(): # @@ -239,9 +234,7 @@ class ArtifactCache(BaseCache): removed = [] modified = [] - self.cas.diff_trees( - digest_a, digest_b, added=added, removed=removed, modified=modified - ) + self.cas.diff_trees(digest_a, digest_b, added=added, removed=removed, modified=modified) return modified, removed, added @@ -271,14 +264,10 @@ class ArtifactCache(BaseCache): # can perform file checks on their end for remote in storage_remotes: remote.init() - element.status( - "Pushing data from artifact {} -> {}".format(display_key, remote) - ) + element.status("Pushing data from artifact {} -> {}".format(display_key, remote)) if self._push_artifact_blobs(artifact, remote): - element.info( - "Pushed data from artifact {} -> {}".format(display_key, remote) - ) + element.info("Pushed data from artifact {} -> {}".format(display_key, remote)) else: element.info( "Remote ({}) already has all data of artifact {} cached".format( @@ -295,9 +284,7 @@ class ArtifactCache(BaseCache): pushed = True else: element.info( - "Remote ({}) already has artifact {} cached".format( - remote, element._get_brief_display_key() - ) + "Remote ({}) already has artifact {} cached".format(remote, element._get_brief_display_key()) ) return pushed @@ -331,19 +318,14 @@ class ArtifactCache(BaseCache): element.info("Pulled artifact {} <- {}".format(display_key, remote)) break else: - element.info( - "Remote ({}) does not have artifact {} cached".format( - remote, display_key - ) - ) + element.info("Remote ({}) does not have artifact {} cached".format(remote, display_key)) except CASError as e: element.warn("Could not pull from remote {}: {}".format(remote, e)) errors.append(e) if errors and not artifact: raise ArtifactError( - "Failed to pull artifact {}".format(display_key), - detail="\n".join(str(e) for e in errors), + "Failed to pull artifact {}".format(display_key), detail="\n".join(str(e) for e in errors), ) # If we don't have an artifact, we can't exactly pull our @@ -356,31 +338,20 @@ class ArtifactCache(BaseCache): for remote in self._storage_remotes[project]: remote.init() try: - element.status( - "Pulling data for artifact {} <- {}".format(display_key, remote) - ) + element.status("Pulling data for artifact {} <- {}".format(display_key, remote)) - if self._pull_artifact_storage( - element, artifact, remote, pull_buildtrees=pull_buildtrees - ): - element.info( - "Pulled data for artifact {} <- {}".format(display_key, remote) - ) + if self._pull_artifact_storage(element, artifact, remote, pull_buildtrees=pull_buildtrees): + element.info("Pulled data for artifact {} <- {}".format(display_key, remote)) return True - element.info( - "Remote ({}) does not have artifact {} cached".format( - remote, display_key - ) - ) + element.info("Remote ({}) does not have artifact {} cached".format(remote, display_key)) except CASError as e: element.warn("Could not pull from remote {}: {}".format(remote, e)) errors.append(e) if errors: raise ArtifactError( - "Failed to pull artifact {}".format(display_key), - detail="\n".join(str(e) for e in errors), + "Failed to pull artifact {}".format(display_key), detail="\n".join(str(e) for e in errors), ) return False @@ -424,8 +395,7 @@ class ArtifactCache(BaseCache): if not push_remotes: raise ArtifactError( - "push_message was called, but no remote artifact " - + "servers are configured as push remotes." + "push_message was called, but no remote artifact " + "servers are configured as push remotes." ) for remote in push_remotes: @@ -448,8 +418,7 @@ class ArtifactCache(BaseCache): if not os.path.exists(os.path.join(self.artifactdir, newref)): os.link( - os.path.join(self.artifactdir, oldref), - os.path.join(self.artifactdir, newref), + os.path.join(self.artifactdir, oldref), os.path.join(self.artifactdir, newref), ) # get_artifact_logs(): @@ -622,15 +591,11 @@ class ArtifactCache(BaseCache): except CASRemoteError as cas_error: if cas_error.reason != "cache-too-full": - raise ArtifactError( - "Failed to push artifact blobs: {}".format(cas_error) - ) + raise ArtifactError("Failed to push artifact blobs: {}".format(cas_error)) return False except grpc.RpcError as e: if e.code() != grpc.StatusCode.RESOURCE_EXHAUSTED: - raise ArtifactError( - "Failed to push artifact blobs: {}".format(e.details()) - ) + raise ArtifactError("Failed to push artifact blobs: {}".format(e.details())) return False return True @@ -655,9 +620,7 @@ class ArtifactCache(BaseCache): artifact_proto = artifact._get_proto() - keys = list( - utils._deduplicate([artifact_proto.strong_key, artifact_proto.weak_key]) - ) + keys = list(utils._deduplicate([artifact_proto.strong_key, artifact_proto.weak_key])) # Check whether the artifact is on the server for key in keys: @@ -665,18 +628,14 @@ class ArtifactCache(BaseCache): remote.get_artifact(element.get_artifact_name(key=key)) except grpc.RpcError as e: if e.code() != grpc.StatusCode.NOT_FOUND: - raise ArtifactError( - "Error checking artifact cache: {}".format(e.details()) - ) + raise ArtifactError("Error checking artifact cache: {}".format(e.details())) else: return False # If not, we send the artifact proto for key in keys: try: - remote.update_artifact( - element.get_artifact_name(key=key), artifact_proto - ) + remote.update_artifact(element.get_artifact_name(key=key), artifact_proto) except grpc.RpcError as e: raise ArtifactError("Failed to push artifact: {}".format(e.details())) |