diff options
author | Tristan Maat <tristan.maat@codethink.co.uk> | 2019-10-16 17:14:15 +0100 |
---|---|---|
committer | Tristan Maat <tristan.maat@codethink.co.uk> | 2019-11-28 14:37:56 +0000 |
commit | 161a1d95f9835e61829fbfe088ded8cb88348be1 (patch) | |
tree | a466bf0cec4d79da58b7001479909b8fd44e6a06 /src/buildstream | |
parent | 75400e85c619c275a4fb1d948edcd846d7ccf613 (diff) | |
download | buildstream-161a1d95f9835e61829fbfe088ded8cb88348be1.tar.gz |
Remove newly unused API surfaces in CASCache
This also involves a number of changes to tests and other parts of the
codebase since they were hacking about wit API that shouldn't have
existed.
Diffstat (limited to 'src/buildstream')
-rw-r--r-- | src/buildstream/_artifactcache.py | 6 | ||||
-rw-r--r-- | src/buildstream/_basecache.py | 25 | ||||
-rw-r--r-- | src/buildstream/_cas/cascache.py | 165 | ||||
-rw-r--r-- | src/buildstream/_exceptions.py | 9 |
4 files changed, 37 insertions, 168 deletions
diff --git a/src/buildstream/_artifactcache.py b/src/buildstream/_artifactcache.py index 03c47b906..430c89610 100644 --- a/src/buildstream/_artifactcache.py +++ b/src/buildstream/_artifactcache.py @@ -22,7 +22,7 @@ import os import grpc from ._basecache import BaseCache -from ._exceptions import ArtifactError, CASError, CASCacheError, CASRemoteError, RemoteError +from ._exceptions import ArtifactError, CASError, CacheError, CASRemoteError, RemoteError from ._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc, artifact_pb2, artifact_pb2_grpc from ._remote import BaseRemote @@ -202,8 +202,8 @@ class ArtifactCache(BaseCache): # def remove(self, ref): try: - self.cas.remove(ref, basedir=self.artifactdir) - except CASCacheError as e: + self._remove_ref(ref, self.artifactdir) + except CacheError as e: raise ArtifactError("{}".format(e)) from e # diff(): diff --git a/src/buildstream/_basecache.py b/src/buildstream/_basecache.py index 516119cd1..2def9a1b1 100644 --- a/src/buildstream/_basecache.py +++ b/src/buildstream/_basecache.py @@ -25,7 +25,7 @@ from . import utils from . import _yaml from ._cas import CASRemote from ._message import Message, MessageType -from ._exceptions import LoadError, RemoteError +from ._exceptions import LoadError, RemoteError, CacheError from ._remote import RemoteSpec, RemoteType @@ -429,3 +429,26 @@ class BaseCache: if not glob_expr or fnmatch(relative_path, glob_expr): # Obtain the mtime (the time a file was last modified) yield (os.path.getmtime(ref_path), relative_path) + + # _remove_ref() + # + # Removes a ref. + # + # This also takes care of pruning away directories which can + # be removed after having removed the given ref. + # + # Args: + # ref (str): The ref to remove + # basedir (str): Path of base directory the ref is in + # + # Raises: + # (CASCacheError): If the ref didnt exist, or a system error + # occurred while removing it + # + def _remove_ref(self, ref, basedir): + try: + utils._remove_path_with_parents(basedir, ref) + except FileNotFoundError as e: + raise CacheError("Could not find ref '{}'".format(ref)) from e + except OSError as e: + raise CacheError("System error while removing ref '{}': {}".format(ref, e)) from e diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py index 98581d351..c45a199fe 100644 --- a/src/buildstream/_cas/cascache.py +++ b/src/buildstream/_cas/cascache.py @@ -21,7 +21,6 @@ import itertools import os import stat -import errno import contextlib import ctypes import multiprocessing @@ -69,7 +68,6 @@ class CASCache: ): self.casdir = os.path.join(path, "cas") self.tmpdir = os.path.join(path, "tmp") - os.makedirs(os.path.join(self.casdir, "refs", "heads"), exist_ok=True) os.makedirs(os.path.join(self.casdir, "objects"), exist_ok=True) os.makedirs(self.tmpdir, exist_ok=True) @@ -134,9 +132,7 @@ class CASCache: # Preflight check. # def preflight(self): - headdir = os.path.join(self.casdir, "refs", "heads") - objdir = os.path.join(self.casdir, "objects") - if not (os.path.isdir(headdir) and os.path.isdir(objdir)): + if not os.path.join(self.casdir, "objects"): raise CASCacheError("CAS repository check failed for '{}'".format(self.casdir)) # close_grpc_channels(): @@ -160,21 +156,6 @@ class CASCache: self._casd_process_manager.release_resources(messenger) self._casd_process_manager = None - # contains(): - # - # Check whether the specified ref is already available in the local CAS cache. - # - # Args: - # ref (str): The ref to check - # - # Returns: True if the ref is in the cache, False otherwise - # - def contains(self, ref): - refpath = self._refpath(ref) - - # This assumes that the repository doesn't have any dangling pointers - return os.path.exists(refpath) - # contains_file(): # # Check whether a digest corresponds to a file which exists in CAS @@ -261,28 +242,6 @@ class CASCache: fullpath = os.path.join(dest, symlinknode.name) os.symlink(symlinknode.target, fullpath) - # diff(): - # - # Return a list of files that have been added or modified between - # the refs described by ref_a and ref_b. - # - # Args: - # ref_a (str): The first ref - # ref_b (str): The second ref - # subdir (str): A subdirectory to limit the comparison to - # - def diff(self, ref_a, ref_b): - tree_a = self.resolve_ref(ref_a) - tree_b = self.resolve_ref(ref_b) - - added = [] - removed = [] - modified = [] - - self.diff_trees(tree_a, tree_b, added=added, removed=removed, modified=modified) - - return modified, removed, added - # pull_tree(): # # Pull a single Tree rather than a ref. @@ -409,74 +368,6 @@ class CASCache: return utils._message_digest(root_directory) - # set_ref(): - # - # Create or replace a ref. - # - # Args: - # ref (str): The name of the ref - # - def set_ref(self, ref, tree): - refpath = self._refpath(ref) - os.makedirs(os.path.dirname(refpath), exist_ok=True) - with utils.save_file_atomic(refpath, "wb", tempdir=self.tmpdir) as f: - f.write(tree.SerializeToString()) - - # resolve_ref(): - # - # Resolve a ref to a digest. - # - # Args: - # ref (str): The name of the ref - # update_mtime (bool): Whether to update the mtime of the ref - # - # Returns: - # (Digest): The digest stored in the ref - # - def resolve_ref(self, ref, *, update_mtime=False): - refpath = self._refpath(ref) - - try: - with open(refpath, "rb") as f: - if update_mtime: - os.utime(refpath) - - digest = remote_execution_pb2.Digest() - digest.ParseFromString(f.read()) - return digest - - except FileNotFoundError as e: - raise CASCacheError("Attempt to access unavailable ref: {}".format(e)) from e - - # update_mtime() - # - # Update the mtime of a ref. - # - # Args: - # ref (str): The ref to update - # - def update_mtime(self, ref): - try: - os.utime(self._refpath(ref)) - except FileNotFoundError as e: - raise CASCacheError("Attempt to access unavailable ref: {}".format(e)) from e - - # remove(): - # - # Removes the given symbolic ref from the repo. - # - # Args: - # ref (str): A symbolic ref - # basedir (str): Path of base directory the ref is in, defaults to - # CAS refs heads - # - def remove(self, ref, *, basedir=None): - - if basedir is None: - basedir = os.path.join(self.casdir, "refs", "heads") - # Remove cache ref - self._remove_ref(ref, basedir) - def update_tree_mtime(self, tree): reachable = set() self._reachable_refs_dir(reachable, tree, update_mtime=True) @@ -645,60 +536,6 @@ class CASCache: # Local Private Methods # ################################################ - def _refpath(self, ref): - return os.path.join(self.casdir, "refs", "heads", ref) - - # _remove_ref() - # - # Removes a ref. - # - # This also takes care of pruning away directories which can - # be removed after having removed the given ref. - # - # Args: - # ref (str): The ref to remove - # basedir (str): Path of base directory the ref is in - # - # Raises: - # (CASCacheError): If the ref didnt exist, or a system error - # occurred while removing it - # - def _remove_ref(self, ref, basedir): - - # Remove the ref itself - refpath = os.path.join(basedir, ref) - - try: - os.unlink(refpath) - except FileNotFoundError as e: - raise CASCacheError("Could not find ref '{}'".format(ref)) from e - - # Now remove any leading directories - - components = list(os.path.split(ref)) - while components: - components.pop() - refdir = os.path.join(basedir, *components) - - # Break out once we reach the base - if refdir == basedir: - break - - try: - os.rmdir(refdir) - except FileNotFoundError: - # The parent directory did not exist, but it's - # parent directory might still be ready to prune - pass - except OSError as e: - if e.errno == errno.ENOTEMPTY: - # The parent directory was not empty, so we - # cannot prune directories beyond this point - break - - # Something went wrong here - raise CASCacheError("System error while removing ref '{}': {}".format(ref, e)) from e - def _get_subdir(self, tree, subdir): head, name = os.path.split(subdir) if head: diff --git a/src/buildstream/_exceptions.py b/src/buildstream/_exceptions.py index ca17577f7..51f542783 100644 --- a/src/buildstream/_exceptions.py +++ b/src/buildstream/_exceptions.py @@ -273,6 +273,15 @@ class SandboxError(BstError): super().__init__(message, detail=detail, domain=ErrorDomain.SANDBOX, reason=reason) +# CacheError +# +# Raised when errors are encountered in either type of cache +# +class CacheError(BstError): + def __init__(self, message, detail=None, reason=None): + super().__init__(message, detail=detail, domain=ErrorDomain.SANDBOX, reason=reason) + + # SourceCacheError # # Raised when errors are encountered in the source caches |