Merge branch 'juerg/fetch-tree' into 'master'

Reimplement contains_directory() with FetchTree()

See merge request BuildStream/buildstream!1642

3 files changed, 20 insertions, 28 deletions

diff --git a/src/buildstream/_artifact.py b/src/buildstream/_artifact.py
index 1e3e39b8f..e5174eaea 100644
--- a/src/buildstream/_artifact.py
+++ b/src/buildstream/_artifact.py
@@ -424,7 +424,7 @@ class Artifact():
 
         # Check whether 'files' subdirectory is available, with or without file contents
         if (require_directories and str(artifact.files) and
-                not self._cas.contains_directory(artifact.files, with_files=require_files, update_mtime=True)):
+                not self._cas.contains_directory(artifact.files, with_files=require_files)):
             self._cached = False
             return False
 
diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py
index 4ee575d05..022730445 100644
--- a/src/buildstream/_cas/cascache.py
+++ b/src/buildstream/_cas/cascache.py
@@ -251,38 +251,25 @@ class CASCache():
     # Args:
     #     digest (Digest): The directory digest to check
     #     with_files (bool): Whether to check files as well
-    #     update_mtime (bool): Whether to update the timestamp
     #
     # Returns: True if the directory is available in the local cache
     #
-    def contains_directory(self, digest, *, with_files, update_mtime=False):
-        try:
-            directory = remote_execution_pb2.Directory()
-            path = self.objpath(digest)
-            with open(path, 'rb') as f:
-                directory.ParseFromString(f.read())
-                if update_mtime:
-                    os.utime(f.fileno())
-
-            # Optionally check presence of files
-            if with_files:
-                for filenode in directory.files:
-                    path = self.objpath(filenode.digest)
-                    if update_mtime:
-                        # No need for separate `exists()` call as this will raise
-                        # FileNotFoundError if the file does not exist.
-                        os.utime(path)
-                    elif not os.path.exists(path):
-                        return False
-
-            # Check subdirectories
-            for dirnode in directory.directories:
-                if not self.contains_directory(dirnode.digest, with_files=with_files, update_mtime=update_mtime):
-                    return False
+    def contains_directory(self, digest, *, with_files):
+        local_cas = self._get_local_cas()
+
+        request = local_cas_pb2.FetchTreeRequest()
+        request.root_digest.CopyFrom(digest)
+        request.fetch_file_blobs = with_files
 
+        try:
+            local_cas.FetchTree(request)
             return True
-        except FileNotFoundError:
-            return False
+        except grpc.RpcError as e:
+            if e.code() == grpc.StatusCode.NOT_FOUND:
+                return False
+            if e.code() == grpc.StatusCode.UNIMPLEMENTED:
+                raise CASCacheError("Unsupported buildbox-casd version: FetchTree unimplemented") from e
+            raise
 
     # checkout():
     #
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index a6b2af110..9a0a71a97 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -267,6 +267,7 @@ class Element(Plugin):
         self.__tracking_scheduled = False       # Sources are scheduled to be tracked
         self.__pull_done = False                # Whether pull was attempted
         self.__cached_successfully = None       # If the Element is known to be successfully cached
+        self.__source_cached = None             # If the sources are known to be successfully cached
         self.__splits = None                    # Resolved regex objects for computing split domains
         self.__whitelist_regex = None           # Resolved regex object to check if file is allowed to overlap
         # Location where Element.stage_sources() was called
@@ -2173,6 +2174,9 @@ class Element(Plugin):
 
     # Check if sources are cached, generating the source key if it hasn't been
     def _source_cached(self):
+        if self.__source_cached is not None:
+            return self.__source_cached
+
         if self.__sources:
             sourcecache = self._get_context().sourcecache
 
@@ -2189,6 +2193,7 @@ class Element(Plugin):
                 if not sourcecache.contains(source):
                     return False
 
+        self.__source_cached = True
         return True
 
     def _should_fetch(self, fetch_original=False):