From 320b3f2efed977bc11903e5c981a17f7a709022c Mon Sep 17 00:00:00 2001 From: Tristan Maat Date: Wed, 16 Oct 2019 17:14:15 +0100 Subject: Remove newly unused API surfaces in CASCache This also involves a number of changes to tests and other parts of the codebase since they were hacking about wit API that shouldn't have existed. --- src/buildstream/_artifactcache.py | 6 +- src/buildstream/_basecache.py | 25 +++++- src/buildstream/_cas/cascache.py | 165 +------------------------------------- src/buildstream/_exceptions.py | 9 +++ tests/sourcecache/fetch.py | 10 +-- tests/sourcecache/push.py | 6 +- tests/testutils/artifactshare.py | 16 +++- 7 files changed, 58 insertions(+), 179 deletions(-) diff --git a/src/buildstream/_artifactcache.py b/src/buildstream/_artifactcache.py index 10ccf1527..3ad288962 100644 --- a/src/buildstream/_artifactcache.py +++ b/src/buildstream/_artifactcache.py @@ -22,7 +22,7 @@ import os import grpc from ._basecache import BaseCache -from ._exceptions import ArtifactError, CASError, CASCacheError, CASRemoteError, RemoteError +from ._exceptions import ArtifactError, CASError, CacheError, CASRemoteError, RemoteError from ._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc, artifact_pb2, artifact_pb2_grpc from ._remote import BaseRemote @@ -202,8 +202,8 @@ class ArtifactCache(BaseCache): # def remove(self, ref): try: - self.cas.remove(ref, basedir=self.artifactdir) - except CASCacheError as e: + self._remove_ref(ref, self.artifactdir) + except CacheError as e: raise ArtifactError("{}".format(e)) from e # diff(): diff --git a/src/buildstream/_basecache.py b/src/buildstream/_basecache.py index 15b1d5389..a7d06f27e 100644 --- a/src/buildstream/_basecache.py +++ b/src/buildstream/_basecache.py @@ -25,7 +25,7 @@ from . import utils from . import _yaml from ._cas import CASRemote from ._message import Message, MessageType -from ._exceptions import LoadError, RemoteError +from ._exceptions import LoadError, RemoteError, CacheError from ._remote import RemoteSpec, RemoteType @@ -429,3 +429,26 @@ class BaseCache: if not glob_expr or fnmatch(relative_path, glob_expr): # Obtain the mtime (the time a file was last modified) yield (os.path.getmtime(ref_path), relative_path) + + # _remove_ref() + # + # Removes a ref. + # + # This also takes care of pruning away directories which can + # be removed after having removed the given ref. + # + # Args: + # ref (str): The ref to remove + # basedir (str): Path of base directory the ref is in + # + # Raises: + # (CASCacheError): If the ref didnt exist, or a system error + # occurred while removing it + # + def _remove_ref(self, ref, basedir): + try: + utils._remove_path_with_parents(basedir, ref) + except FileNotFoundError as e: + raise CacheError("Could not find ref '{}'".format(ref)) from e + except OSError as e: + raise CacheError("System error while removing ref '{}': {}".format(ref, e)) from e diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py index 98581d351..c45a199fe 100644 --- a/src/buildstream/_cas/cascache.py +++ b/src/buildstream/_cas/cascache.py @@ -21,7 +21,6 @@ import itertools import os import stat -import errno import contextlib import ctypes import multiprocessing @@ -69,7 +68,6 @@ class CASCache: ): self.casdir = os.path.join(path, "cas") self.tmpdir = os.path.join(path, "tmp") - os.makedirs(os.path.join(self.casdir, "refs", "heads"), exist_ok=True) os.makedirs(os.path.join(self.casdir, "objects"), exist_ok=True) os.makedirs(self.tmpdir, exist_ok=True) @@ -134,9 +132,7 @@ class CASCache: # Preflight check. # def preflight(self): - headdir = os.path.join(self.casdir, "refs", "heads") - objdir = os.path.join(self.casdir, "objects") - if not (os.path.isdir(headdir) and os.path.isdir(objdir)): + if not os.path.join(self.casdir, "objects"): raise CASCacheError("CAS repository check failed for '{}'".format(self.casdir)) # close_grpc_channels(): @@ -160,21 +156,6 @@ class CASCache: self._casd_process_manager.release_resources(messenger) self._casd_process_manager = None - # contains(): - # - # Check whether the specified ref is already available in the local CAS cache. - # - # Args: - # ref (str): The ref to check - # - # Returns: True if the ref is in the cache, False otherwise - # - def contains(self, ref): - refpath = self._refpath(ref) - - # This assumes that the repository doesn't have any dangling pointers - return os.path.exists(refpath) - # contains_file(): # # Check whether a digest corresponds to a file which exists in CAS @@ -261,28 +242,6 @@ class CASCache: fullpath = os.path.join(dest, symlinknode.name) os.symlink(symlinknode.target, fullpath) - # diff(): - # - # Return a list of files that have been added or modified between - # the refs described by ref_a and ref_b. - # - # Args: - # ref_a (str): The first ref - # ref_b (str): The second ref - # subdir (str): A subdirectory to limit the comparison to - # - def diff(self, ref_a, ref_b): - tree_a = self.resolve_ref(ref_a) - tree_b = self.resolve_ref(ref_b) - - added = [] - removed = [] - modified = [] - - self.diff_trees(tree_a, tree_b, added=added, removed=removed, modified=modified) - - return modified, removed, added - # pull_tree(): # # Pull a single Tree rather than a ref. @@ -409,74 +368,6 @@ class CASCache: return utils._message_digest(root_directory) - # set_ref(): - # - # Create or replace a ref. - # - # Args: - # ref (str): The name of the ref - # - def set_ref(self, ref, tree): - refpath = self._refpath(ref) - os.makedirs(os.path.dirname(refpath), exist_ok=True) - with utils.save_file_atomic(refpath, "wb", tempdir=self.tmpdir) as f: - f.write(tree.SerializeToString()) - - # resolve_ref(): - # - # Resolve a ref to a digest. - # - # Args: - # ref (str): The name of the ref - # update_mtime (bool): Whether to update the mtime of the ref - # - # Returns: - # (Digest): The digest stored in the ref - # - def resolve_ref(self, ref, *, update_mtime=False): - refpath = self._refpath(ref) - - try: - with open(refpath, "rb") as f: - if update_mtime: - os.utime(refpath) - - digest = remote_execution_pb2.Digest() - digest.ParseFromString(f.read()) - return digest - - except FileNotFoundError as e: - raise CASCacheError("Attempt to access unavailable ref: {}".format(e)) from e - - # update_mtime() - # - # Update the mtime of a ref. - # - # Args: - # ref (str): The ref to update - # - def update_mtime(self, ref): - try: - os.utime(self._refpath(ref)) - except FileNotFoundError as e: - raise CASCacheError("Attempt to access unavailable ref: {}".format(e)) from e - - # remove(): - # - # Removes the given symbolic ref from the repo. - # - # Args: - # ref (str): A symbolic ref - # basedir (str): Path of base directory the ref is in, defaults to - # CAS refs heads - # - def remove(self, ref, *, basedir=None): - - if basedir is None: - basedir = os.path.join(self.casdir, "refs", "heads") - # Remove cache ref - self._remove_ref(ref, basedir) - def update_tree_mtime(self, tree): reachable = set() self._reachable_refs_dir(reachable, tree, update_mtime=True) @@ -645,60 +536,6 @@ class CASCache: # Local Private Methods # ################################################ - def _refpath(self, ref): - return os.path.join(self.casdir, "refs", "heads", ref) - - # _remove_ref() - # - # Removes a ref. - # - # This also takes care of pruning away directories which can - # be removed after having removed the given ref. - # - # Args: - # ref (str): The ref to remove - # basedir (str): Path of base directory the ref is in - # - # Raises: - # (CASCacheError): If the ref didnt exist, or a system error - # occurred while removing it - # - def _remove_ref(self, ref, basedir): - - # Remove the ref itself - refpath = os.path.join(basedir, ref) - - try: - os.unlink(refpath) - except FileNotFoundError as e: - raise CASCacheError("Could not find ref '{}'".format(ref)) from e - - # Now remove any leading directories - - components = list(os.path.split(ref)) - while components: - components.pop() - refdir = os.path.join(basedir, *components) - - # Break out once we reach the base - if refdir == basedir: - break - - try: - os.rmdir(refdir) - except FileNotFoundError: - # The parent directory did not exist, but it's - # parent directory might still be ready to prune - pass - except OSError as e: - if e.errno == errno.ENOTEMPTY: - # The parent directory was not empty, so we - # cannot prune directories beyond this point - break - - # Something went wrong here - raise CASCacheError("System error while removing ref '{}': {}".format(ref, e)) from e - def _get_subdir(self, tree, subdir): head, name = os.path.split(subdir) if head: diff --git a/src/buildstream/_exceptions.py b/src/buildstream/_exceptions.py index ca17577f7..51f542783 100644 --- a/src/buildstream/_exceptions.py +++ b/src/buildstream/_exceptions.py @@ -273,6 +273,15 @@ class SandboxError(BstError): super().__init__(message, detail=detail, domain=ErrorDomain.SANDBOX, reason=reason) +# CacheError +# +# Raised when errors are encountered in either type of cache +# +class CacheError(BstError): + def __init__(self, message, detail=None, reason=None): + super().__init__(message, detail=detail, domain=ErrorDomain.SANDBOX, reason=reason) + + # SourceCacheError # # Raised when errors are encountered in the source caches diff --git a/tests/sourcecache/fetch.py b/tests/sourcecache/fetch.py index 0c347ebbf..4096b56b8 100644 --- a/tests/sourcecache/fetch.py +++ b/tests/sourcecache/fetch.py @@ -37,6 +37,7 @@ DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") def move_local_cas_to_remote_source_share(local, remote): shutil.rmtree(os.path.join(remote, "repo", "cas")) + shutil.rmtree(os.path.join(remote, "repo", "source_protos")) shutil.move(os.path.join(local, "source_protos"), os.path.join(remote, "repo")) shutil.move(os.path.join(local, "cas"), os.path.join(remote, "repo")) shutil.rmtree(os.path.join(local, "sources")) @@ -85,8 +86,7 @@ def test_source_fetch(cli, tmpdir, datafiles): assert not element._source_cached() source = list(element.sources())[0] - cas = context.get_cascache() - assert not cas.contains(source._get_source_name()) + assert not share.get_source_proto(source._get_source_name()) # Just check that we sensibly fetch and build the element res = cli.run(project=project_dir, args=["build", element_name]) @@ -132,8 +132,7 @@ def test_fetch_fallback(cli, tmpdir, datafiles): assert not element._source_cached() source = list(element.sources())[0] - cas = context.get_cascache() - assert not cas.contains(source._get_source_name()) + assert not share.get_source_proto(source._get_source_name()) assert not os.path.exists(os.path.join(cache_dir, "sources")) # Now check if it falls back to the source fetch method. @@ -195,8 +194,7 @@ def test_source_pull_partial_fallback_fetch(cli, tmpdir, datafiles): assert not element._source_cached() source = list(element.sources())[0] - cas = context.get_cascache() - assert not cas.contains(source._get_source_name()) + assert not share.get_artifact_proto(source._get_source_name()) # Just check that we sensibly fetch and build the element res = cli.run(project=project_dir, args=["build", element_name]) diff --git a/tests/sourcecache/push.py b/tests/sourcecache/push.py index 719860425..0b7bb9954 100644 --- a/tests/sourcecache/push.py +++ b/tests/sourcecache/push.py @@ -89,8 +89,7 @@ def test_source_push_split(cli, tmpdir, datafiles): source = list(element.sources())[0] # check we don't have it in the current cache - cas = context.get_cascache() - assert not cas.contains(source._get_source_name()) + assert not index.get_source_proto(source._get_source_name()) # build the element, this should fetch and then push the source to the # remote @@ -139,8 +138,7 @@ def test_source_push(cli, tmpdir, datafiles): source = list(element.sources())[0] # check we don't have it in the current cache - cas = context.get_cascache() - assert not cas.contains(source._get_source_name()) + assert not share.get_source_proto(source._get_source_name()) # build the element, this should fetch and then push the source to the # remote diff --git a/tests/testutils/artifactshare.py b/tests/testutils/artifactshare.py index 8d0448f8a..19c19131a 100644 --- a/tests/testutils/artifactshare.py +++ b/tests/testutils/artifactshare.py @@ -13,7 +13,7 @@ from buildstream._cas import CASCache from buildstream._cas.casserver import create_server from buildstream._exceptions import CASError from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 -from buildstream._protos.buildstream.v2 import artifact_pb2 +from buildstream._protos.buildstream.v2 import artifact_pb2, source_pb2 class BaseArtifactShare: @@ -120,6 +120,8 @@ class ArtifactShare(BaseArtifactShare): os.makedirs(self.repodir) self.artifactdir = os.path.join(self.repodir, "artifacts", "refs") os.makedirs(self.artifactdir) + self.sourcedir = os.path.join(self.repodir, "source_protos", "refs") + os.makedirs(self.sourcedir) self.cas = CASCache(self.repodir, casd=casd) @@ -160,6 +162,18 @@ class ArtifactShare(BaseArtifactShare): return artifact_proto + def get_source_proto(self, source_name): + source_proto = source_pb2.Source() + source_path = os.path.join(self.sourcedir, source_name) + + try: + with open(source_path, "rb") as f: + source_proto.ParseFromString(f.read()) + except FileNotFoundError: + return None + + return source_proto + def get_cas_files(self, artifact_proto): reachable = set() -- cgit v1.2.1