Update mtimes of objects for requested references.

This also remove references when some objects are missing. This is in
preparation for the move from reference to object garbage collection.

3 files changed, 37 insertions, 11 deletions

diff --git a/buildstream/_artifactcache/cascache.py b/buildstream/_artifactcache/cascache.py
index 1f3f975b7..905a6cc7d 100644
--- a/buildstream/_artifactcache/cascache.py
+++ b/buildstream/_artifactcache/cascache.py
@@ -577,6 +577,10 @@ class CASCache(ArtifactCache):
 
         return pruned
 
+    def update_tree_mtime(self, tree):
+        reachable = set()
+        self._reachable_refs_dir(reachable, tree, update_mtime=True)
+
     ################################################
     #             Local Private Methods            #
     ################################################
@@ -718,10 +722,13 @@ class CASCache(ArtifactCache):
                 a += 1
                 b += 1
 
-    def _reachable_refs_dir(self, reachable, tree):
+    def _reachable_refs_dir(self, reachable, tree, update_mtime=False):
         if tree.hash in reachable:
             return
 
+        if update_mtime:
+            os.utime(self.objpath(tree))
+
         reachable.add(tree.hash)
 
         directory = remote_execution_pb2.Directory()
@@ -730,10 +737,12 @@ class CASCache(ArtifactCache):
             directory.ParseFromString(f.read())
 
         for filenode in directory.files:
+            if update_mtime:
+                os.utime(self.objpath(filenode.digest))
             reachable.add(filenode.digest.hash)
 
         for dirnode in directory.directories:
-            self._reachable_refs_dir(reachable, dirnode.digest)
+            self._reachable_refs_dir(reachable, dirnode.digest, update_mtime=update_mtime)
 
     def _initialize_remote(self, remote_spec, q):
         try:
diff --git a/buildstream/_artifactcache/casserver.py b/buildstream/_artifactcache/casserver.py
index c99ef888e..df9c4740c 100644
--- a/buildstream/_artifactcache/casserver.py
+++ b/buildstream/_artifactcache/casserver.py
@@ -250,10 +250,17 @@ class _ContentAddressableStorageServicer(remote_execution_pb2_grpc.ContentAddres
     def FindMissingBlobs(self, request, context):
         response = remote_execution_pb2.FindMissingBlobsResponse()
         for digest in request.blob_digests:
-            if not _has_object(self.cas, digest):
-                d = response.missing_blob_digests.add()
-                d.hash = digest.hash
-                d.size_bytes = digest.size_bytes
+            objpath = self.cas.objpath(digest)
+            try:
+                os.utime(objpath)
+            except OSError as e:
+                if e.errno != errno.ENOENT:
+                    raise
+                else:
+                    d = response.missing_blob_digests.add()
+                    d.hash = digest.hash
+                    d.size_bytes = digest.size_bytes
+
         return response
 
     def BatchReadBlobs(self, request, context):
@@ -350,6 +357,12 @@ class _ReferenceStorageServicer(buildstream_pb2_grpc.ReferenceStorageServicer):
 
         try:
             tree = self.cas.resolve_ref(request.key, update_mtime=True)
+            try:
+                self.cas.update_tree_mtime(tree)
+            except FileNotFoundError:
+                self.cas.remove(request.key, defer_prune=True)
+                context.set_code(grpc.StatusCode.NOT_FOUND)
+                return response
 
             response.digest.hash = tree.hash
             response.digest.size_bytes = tree.size_bytes
@@ -422,11 +435,6 @@ def _digest_from_upload_resource_name(resource_name):
         return None
 
 
-def _has_object(cas, digest):
-    objpath = cas.objpath(digest)
-    return os.path.exists(objpath)
-
-
 # _clean_up_cache()
 #
 # Keep removing Least Recently Pushed (LRP) artifacts in a cache until there
diff --git a/tests/testutils/artifactshare.py b/tests/testutils/artifactshare.py
index 05e87a499..3319cb0a9 100644
--- a/tests/testutils/artifactshare.py
+++ b/tests/testutils/artifactshare.py
@@ -118,6 +118,15 @@ class ArtifactShare():
 
         try:
             tree = self.cas.resolve_ref(artifact_key)
+            reachable = set()
+            try:
+                self.cas._reachable_refs_dir(reachable, tree, update_mtime=False)
+            except FileNotFoundError:
+                return False
+            for digest in reachable:
+                object_name = os.path.join(self.cas.casdir, 'objects', digest[:2], digest[2:])
+                if not os.path.exists(object_name):
+                    return False
             return True
         except ArtifactError:
             return False