Remove newly unused API surfaces in CASCache

This also involves a number of changes to tests and other parts of the
codebase since they were hacking about wit API that shouldn't have
existed.

7 files changed, 58 insertions, 179 deletions

diff --git a/src/buildstream/_artifactcache.py b/src/buildstream/_artifactcache.py
index 10ccf1527..3ad288962 100644
--- a/src/buildstream/_artifactcache.py
+++ b/src/buildstream/_artifactcache.py
@@ -22,7 +22,7 @@ import os
 import grpc
 
 from ._basecache import BaseCache
-from ._exceptions import ArtifactError, CASError, CASCacheError, CASRemoteError, RemoteError
+from ._exceptions import ArtifactError, CASError, CacheError, CASRemoteError, RemoteError
 from ._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc, artifact_pb2, artifact_pb2_grpc
 
 from ._remote import BaseRemote
@@ -202,8 +202,8 @@ class ArtifactCache(BaseCache):
     #
     def remove(self, ref):
         try:
-            self.cas.remove(ref, basedir=self.artifactdir)
-        except CASCacheError as e:
+            self._remove_ref(ref, self.artifactdir)
+        except CacheError as e:
             raise ArtifactError("{}".format(e)) from e
 
     # diff():
diff --git a/src/buildstream/_basecache.py b/src/buildstream/_basecache.py
index 15b1d5389..a7d06f27e 100644
--- a/src/buildstream/_basecache.py
+++ b/src/buildstream/_basecache.py
@@ -25,7 +25,7 @@ from . import utils
 from . import _yaml
 from ._cas import CASRemote
 from ._message import Message, MessageType
-from ._exceptions import LoadError, RemoteError
+from ._exceptions import LoadError, RemoteError, CacheError
 from ._remote import RemoteSpec, RemoteType
 
 
@@ -429,3 +429,26 @@ class BaseCache:
                 if not glob_expr or fnmatch(relative_path, glob_expr):
                     # Obtain the mtime (the time a file was last modified)
                     yield (os.path.getmtime(ref_path), relative_path)
+
+    # _remove_ref()
+    #
+    # Removes a ref.
+    #
+    # This also takes care of pruning away directories which can
+    # be removed after having removed the given ref.
+    #
+    # Args:
+    #    ref (str): The ref to remove
+    #    basedir (str): Path of base directory the ref is in
+    #
+    # Raises:
+    #    (CASCacheError): If the ref didnt exist, or a system error
+    #                     occurred while removing it
+    #
+    def _remove_ref(self, ref, basedir):
+        try:
+            utils._remove_path_with_parents(basedir, ref)
+        except FileNotFoundError as e:
+            raise CacheError("Could not find ref '{}'".format(ref)) from e
+        except OSError as e:
+            raise CacheError("System error while removing ref '{}': {}".format(ref, e)) from e
diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py
index 98581d351..c45a199fe 100644
--- a/src/buildstream/_cas/cascache.py
+++ b/src/buildstream/_cas/cascache.py
@@ -21,7 +21,6 @@
 import itertools
 import os
 import stat
-import errno
 import contextlib
 import ctypes
 import multiprocessing
@@ -69,7 +68,6 @@ class CASCache:
     ):
         self.casdir = os.path.join(path, "cas")
         self.tmpdir = os.path.join(path, "tmp")
-        os.makedirs(os.path.join(self.casdir, "refs", "heads"), exist_ok=True)
         os.makedirs(os.path.join(self.casdir, "objects"), exist_ok=True)
         os.makedirs(self.tmpdir, exist_ok=True)
 
@@ -134,9 +132,7 @@ class CASCache:
     # Preflight check.
     #
     def preflight(self):
-        headdir = os.path.join(self.casdir, "refs", "heads")
-        objdir = os.path.join(self.casdir, "objects")
-        if not (os.path.isdir(headdir) and os.path.isdir(objdir)):
+        if not os.path.join(self.casdir, "objects"):
             raise CASCacheError("CAS repository check failed for '{}'".format(self.casdir))
 
     # close_grpc_channels():
@@ -160,21 +156,6 @@ class CASCache:
             self._casd_process_manager.release_resources(messenger)
             self._casd_process_manager = None
 
-    # contains():
-    #
-    # Check whether the specified ref is already available in the local CAS cache.
-    #
-    # Args:
-    #     ref (str): The ref to check
-    #
-    # Returns: True if the ref is in the cache, False otherwise
-    #
-    def contains(self, ref):
-        refpath = self._refpath(ref)
-
-        # This assumes that the repository doesn't have any dangling pointers
-        return os.path.exists(refpath)
-
     # contains_file():
     #
     # Check whether a digest corresponds to a file which exists in CAS
@@ -261,28 +242,6 @@ class CASCache:
             fullpath = os.path.join(dest, symlinknode.name)
             os.symlink(symlinknode.target, fullpath)
 
-    # diff():
-    #
-    # Return a list of files that have been added or modified between
-    # the refs described by ref_a and ref_b.
-    #
-    # Args:
-    #     ref_a (str): The first ref
-    #     ref_b (str): The second ref
-    #     subdir (str): A subdirectory to limit the comparison to
-    #
-    def diff(self, ref_a, ref_b):
-        tree_a = self.resolve_ref(ref_a)
-        tree_b = self.resolve_ref(ref_b)
-
-        added = []
-        removed = []
-        modified = []
-
-        self.diff_trees(tree_a, tree_b, added=added, removed=removed, modified=modified)
-
-        return modified, removed, added
-
     # pull_tree():
     #
     # Pull a single Tree rather than a ref.
@@ -409,74 +368,6 @@ class CASCache:
 
         return utils._message_digest(root_directory)
 
-    # set_ref():
-    #
-    # Create or replace a ref.
-    #
-    # Args:
-    #     ref (str): The name of the ref
-    #
-    def set_ref(self, ref, tree):
-        refpath = self._refpath(ref)
-        os.makedirs(os.path.dirname(refpath), exist_ok=True)
-        with utils.save_file_atomic(refpath, "wb", tempdir=self.tmpdir) as f:
-            f.write(tree.SerializeToString())
-
-    # resolve_ref():
-    #
-    # Resolve a ref to a digest.
-    #
-    # Args:
-    #     ref (str): The name of the ref
-    #     update_mtime (bool): Whether to update the mtime of the ref
-    #
-    # Returns:
-    #     (Digest): The digest stored in the ref
-    #
-    def resolve_ref(self, ref, *, update_mtime=False):
-        refpath = self._refpath(ref)
-
-        try:
-            with open(refpath, "rb") as f:
-                if update_mtime:
-                    os.utime(refpath)
-
-                digest = remote_execution_pb2.Digest()
-                digest.ParseFromString(f.read())
-                return digest
-
-        except FileNotFoundError as e:
-            raise CASCacheError("Attempt to access unavailable ref: {}".format(e)) from e
-
-    # update_mtime()
-    #
-    # Update the mtime of a ref.
-    #
-    # Args:
-    #     ref (str): The ref to update
-    #
-    def update_mtime(self, ref):
-        try:
-            os.utime(self._refpath(ref))
-        except FileNotFoundError as e:
-            raise CASCacheError("Attempt to access unavailable ref: {}".format(e)) from e
-
-    # remove():
-    #
-    # Removes the given symbolic ref from the repo.
-    #
-    # Args:
-    #    ref (str): A symbolic ref
-    #    basedir (str): Path of base directory the ref is in, defaults to
-    #                   CAS refs heads
-    #
-    def remove(self, ref, *, basedir=None):
-
-        if basedir is None:
-            basedir = os.path.join(self.casdir, "refs", "heads")
-        # Remove cache ref
-        self._remove_ref(ref, basedir)
-
     def update_tree_mtime(self, tree):
         reachable = set()
         self._reachable_refs_dir(reachable, tree, update_mtime=True)
@@ -645,60 +536,6 @@ class CASCache:
     #             Local Private Methods            #
     ################################################
 
-    def _refpath(self, ref):
-        return os.path.join(self.casdir, "refs", "heads", ref)
-
-    # _remove_ref()
-    #
-    # Removes a ref.
-    #
-    # This also takes care of pruning away directories which can
-    # be removed after having removed the given ref.
-    #
-    # Args:
-    #    ref (str): The ref to remove
-    #    basedir (str): Path of base directory the ref is in
-    #
-    # Raises:
-    #    (CASCacheError): If the ref didnt exist, or a system error
-    #                     occurred while removing it
-    #
-    def _remove_ref(self, ref, basedir):
-
-        # Remove the ref itself
-        refpath = os.path.join(basedir, ref)
-
-        try:
-            os.unlink(refpath)
-        except FileNotFoundError as e:
-            raise CASCacheError("Could not find ref '{}'".format(ref)) from e
-
-        # Now remove any leading directories
-
-        components = list(os.path.split(ref))
-        while components:
-            components.pop()
-            refdir = os.path.join(basedir, *components)
-
-            # Break out once we reach the base
-            if refdir == basedir:
-                break
-
-            try:
-                os.rmdir(refdir)
-            except FileNotFoundError:
-                # The parent directory did not exist, but it's
-                # parent directory might still be ready to prune
-                pass
-            except OSError as e:
-                if e.errno == errno.ENOTEMPTY:
-                    # The parent directory was not empty, so we
-                    # cannot prune directories beyond this point
-                    break
-
-                # Something went wrong here
-                raise CASCacheError("System error while removing ref '{}': {}".format(ref, e)) from e
-
     def _get_subdir(self, tree, subdir):
         head, name = os.path.split(subdir)
         if head:
diff --git a/src/buildstream/_exceptions.py b/src/buildstream/_exceptions.py
index ca17577f7..51f542783 100644
--- a/src/buildstream/_exceptions.py
+++ b/src/buildstream/_exceptions.py
@@ -273,6 +273,15 @@ class SandboxError(BstError):
         super().__init__(message, detail=detail, domain=ErrorDomain.SANDBOX, reason=reason)
 
 
+# CacheError
+#
+# Raised when errors are encountered in either type of cache
+#
+class CacheError(BstError):
+    def __init__(self, message, detail=None, reason=None):
+        super().__init__(message, detail=detail, domain=ErrorDomain.SANDBOX, reason=reason)
+
+
 # SourceCacheError
 #
 # Raised when errors are encountered in the source caches
diff --git a/tests/sourcecache/fetch.py b/tests/sourcecache/fetch.py
index 0c347ebbf..4096b56b8 100644
--- a/tests/sourcecache/fetch.py
+++ b/tests/sourcecache/fetch.py
@@ -37,6 +37,7 @@ DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
 
 def move_local_cas_to_remote_source_share(local, remote):
     shutil.rmtree(os.path.join(remote, "repo", "cas"))
+    shutil.rmtree(os.path.join(remote, "repo", "source_protos"))
     shutil.move(os.path.join(local, "source_protos"), os.path.join(remote, "repo"))
     shutil.move(os.path.join(local, "cas"), os.path.join(remote, "repo"))
     shutil.rmtree(os.path.join(local, "sources"))
@@ -85,8 +86,7 @@ def test_source_fetch(cli, tmpdir, datafiles):
             assert not element._source_cached()
             source = list(element.sources())[0]
 
-            cas = context.get_cascache()
-            assert not cas.contains(source._get_source_name())
+            assert not share.get_source_proto(source._get_source_name())
 
             # Just check that we sensibly fetch and build the element
             res = cli.run(project=project_dir, args=["build", element_name])
@@ -132,8 +132,7 @@ def test_fetch_fallback(cli, tmpdir, datafiles):
             assert not element._source_cached()
             source = list(element.sources())[0]
 
-            cas = context.get_cascache()
-            assert not cas.contains(source._get_source_name())
+            assert not share.get_source_proto(source._get_source_name())
             assert not os.path.exists(os.path.join(cache_dir, "sources"))
 
             # Now check if it falls back to the source fetch method.
@@ -195,8 +194,7 @@ def test_source_pull_partial_fallback_fetch(cli, tmpdir, datafiles):
             assert not element._source_cached()
             source = list(element.sources())[0]
 
-            cas = context.get_cascache()
-            assert not cas.contains(source._get_source_name())
+            assert not share.get_artifact_proto(source._get_source_name())
 
             # Just check that we sensibly fetch and build the element
             res = cli.run(project=project_dir, args=["build", element_name])
diff --git a/tests/sourcecache/push.py b/tests/sourcecache/push.py
index 719860425..0b7bb9954 100644
--- a/tests/sourcecache/push.py
+++ b/tests/sourcecache/push.py
@@ -89,8 +89,7 @@ def test_source_push_split(cli, tmpdir, datafiles):
             source = list(element.sources())[0]
 
             # check we don't have it in the current cache
-            cas = context.get_cascache()
-            assert not cas.contains(source._get_source_name())
+            assert not index.get_source_proto(source._get_source_name())
 
             # build the element, this should fetch and then push the source to the
             # remote
@@ -139,8 +138,7 @@ def test_source_push(cli, tmpdir, datafiles):
             source = list(element.sources())[0]
 
             # check we don't have it in the current cache
-            cas = context.get_cascache()
-            assert not cas.contains(source._get_source_name())
+            assert not share.get_source_proto(source._get_source_name())
 
             # build the element, this should fetch and then push the source to the
             # remote
diff --git a/tests/testutils/artifactshare.py b/tests/testutils/artifactshare.py
index 8d0448f8a..19c19131a 100644
--- a/tests/testutils/artifactshare.py
+++ b/tests/testutils/artifactshare.py
@@ -13,7 +13,7 @@ from buildstream._cas import CASCache
 from buildstream._cas.casserver import create_server
 from buildstream._exceptions import CASError
 from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
-from buildstream._protos.buildstream.v2 import artifact_pb2
+from buildstream._protos.buildstream.v2 import artifact_pb2, source_pb2
 
 
 class BaseArtifactShare:
@@ -120,6 +120,8 @@ class ArtifactShare(BaseArtifactShare):
         os.makedirs(self.repodir)
         self.artifactdir = os.path.join(self.repodir, "artifacts", "refs")
         os.makedirs(self.artifactdir)
+        self.sourcedir = os.path.join(self.repodir, "source_protos", "refs")
+        os.makedirs(self.sourcedir)
 
         self.cas = CASCache(self.repodir, casd=casd)
 
@@ -160,6 +162,18 @@ class ArtifactShare(BaseArtifactShare):
 
         return artifact_proto
 
+    def get_source_proto(self, source_name):
+        source_proto = source_pb2.Source()
+        source_path = os.path.join(self.sourcedir, source_name)
+
+        try:
+            with open(source_path, "rb") as f:
+                source_proto.ParseFromString(f.read())
+        except FileNotFoundError:
+            return None
+
+        return source_proto
+
     def get_cas_files(self, artifact_proto):
 
         reachable = set()