Merge branch 'juerg/artifact-blobs' into 'master'

Artifact blob handling

See merge request BuildStream/buildstream!1527

6 files changed, 42 insertions, 27 deletions

diff --git a/src/buildstream/_artifact.py b/src/buildstream/_artifact.py
index 493ca5d26..5499b0dab 100644
--- a/src/buildstream/_artifact.py
+++ b/src/buildstream/_artifact.py
@@ -363,7 +363,7 @@ class Artifact():
 
         # Check whether 'files' subdirectory is available, with or without file contents
         if (require_directories and str(artifact.files) and
-                not self._cas.contains_directory(artifact.files, with_files=require_files)):
+                not self._cas.contains_directory(artifact.files, with_files=require_files, update_mtime=True)):
             self._cached = False
             return False
 
diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py
index dbdfa41a8..535d9199e 100644
--- a/src/buildstream/_cas/cascache.py
+++ b/src/buildstream/_cas/cascache.py
@@ -124,24 +124,33 @@ class CASCache():
     # Args:
     #     digest (Digest): The directory digest to check
     #     with_files (bool): Whether to check files as well
+    #     update_mtime (bool): Whether to update the timestamp
     #
     # Returns: True if the directory is available in the local cache
     #
-    def contains_directory(self, digest, *, with_files):
+    def contains_directory(self, digest, *, with_files, update_mtime=False):
         try:
             directory = remote_execution_pb2.Directory()
-            with open(self.objpath(digest), 'rb') as f:
+            path = self.objpath(digest)
+            with open(path, 'rb') as f:
                 directory.ParseFromString(f.read())
+                if update_mtime:
+                    os.utime(f.fileno())
 
             # Optionally check presence of files
             if with_files:
                 for filenode in directory.files:
-                    if not os.path.exists(self.objpath(filenode.digest)):
+                    path = self.objpath(filenode.digest)
+                    if update_mtime:
+                        # No need for separate `exists()` call as this will raise
+                        # FileNotFoundError if the file does not exist.
+                        os.utime(path)
+                    elif not os.path.exists(path):
                         return False
 
             # Check subdirectories
             for dirnode in directory.directories:
-                if not self.contains_directory(dirnode.digest, with_files=with_files):
+                if not self.contains_directory(dirnode.digest, with_files=with_files, update_mtime=update_mtime):
                     return False
 
             return True
@@ -792,6 +801,9 @@ class CASCache():
                 directory.ParseFromString(f.read())
 
         except FileNotFoundError:
+            if check_exists:
+                raise
+
             # Just exit early if the file doesn't exist
             return
 
@@ -961,6 +973,8 @@ class CASCache():
     def fetch_blobs(self, remote, digests):
         missing_blobs = []
 
+        remote.init()
+
         batch = _CASBatchRead(remote)
 
         for digest in digests:
diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py
index 276f81a6a..81ee4ad83 100644
--- a/src/buildstream/_frontend/cli.py
+++ b/src/buildstream/_frontend/cli.py
@@ -496,9 +496,6 @@ def show(app, elements, deps, except_, order, format_):
             $'---------- %{name} ----------\\n%{vars}'
     """
     with app.initialized():
-        # Do not require artifact directory trees or file contents to be present for `bst show`
-        app.context.set_artifact_directories_optional()
-
         if not elements:
             elements = app.project.get_default_targets()
 
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 21c38bc1a..0d80113ab 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -1703,6 +1703,7 @@ class Element(Plugin):
             try:
                 sandbox_build_dir = sandbox_vroot.descend(
                     *self.get_variable('build-root').lstrip(os.sep).split(os.sep))
+                sandbox._fetch_missing_blobs(sandbox_build_dir)
             except VirtualDirectoryError:
                 # Directory could not be found. Pre-virtual
                 # directory behaviour was to continue silently
@@ -1712,6 +1713,7 @@ class Element(Plugin):
         if collect is not None:
             try:
                 collectvdir = sandbox_vroot.descend(*collect.lstrip(os.sep).split(os.sep))
+                sandbox._fetch_missing_blobs(collectvdir)
             except VirtualDirectoryError:
                 pass
 
diff --git a/src/buildstream/sandbox/_sandboxremote.py b/src/buildstream/sandbox/_sandboxremote.py
index c84bfa4ef..8c9515497 100644
--- a/src/buildstream/sandbox/_sandboxremote.py
+++ b/src/buildstream/sandbox/_sandboxremote.py
@@ -30,7 +30,6 @@ from .. import utils
 from ..node import Node
 from .._message import Message, MessageType
 from .sandbox import Sandbox, SandboxCommandError, _SandboxBatch
-from ..storage.directory import VirtualDirectoryError
 from ..storage._casbaseddirectory import CasBasedDirectory
 from .. import _signals
 from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
@@ -280,7 +279,6 @@ class SandboxRemote(Sandbox):
             raise SandboxError("Output directory structure had no digest attached.")
 
         context = self._get_context()
-        project = self._get_project()
         cascache = context.get_cascache()
         artifactcache = context.artifactcache
         casremote = CASRemote(self.storage_remote_spec)
@@ -294,27 +292,20 @@ class SandboxRemote(Sandbox):
         # to replace the sandbox's virtual directory with that. Creating a new virtual directory object
         # from another hash will be interesting, though...
 
-        new_dir = CasBasedDirectory(context.artifactcache.cas, digest=dir_digest)
+        new_dir = CasBasedDirectory(artifactcache.cas, digest=dir_digest)
         self._set_virtual_directory(new_dir)
 
+    def _fetch_missing_blobs(self, vdir):
+        context = self._get_context()
+        project = self._get_project()
+        cascache = context.get_cascache()
+        artifactcache = context.artifactcache
+        casremote = CASRemote(self.storage_remote_spec)
+
         # Fetch the file blobs if needed
         if self._output_files_required or artifactcache.has_push_remotes():
-            required_blobs = []
-            directories = []
-
-            directories.append(self._output_directory)
-            if self._build_directory and (self._build_directory_always or failure):
-                directories.append(self._build_directory)
-
-            for directory in directories:
-                try:
-                    vdir = new_dir.descend(*directory.strip(os.sep).split(os.sep))
-                    dir_digest = vdir._get_digest()
-                    required_blobs += cascache.required_blobs_for_directory(dir_digest)
-                except VirtualDirectoryError:
-                    # If the directory does not exist, there is no need to
-                    # download file blobs.
-                    pass
+            dir_digest = vdir._get_digest()
+            required_blobs = cascache.required_blobs_for_directory(dir_digest)
 
             local_missing_blobs = cascache.local_missing_blobs(required_blobs)
             if local_missing_blobs:
diff --git a/src/buildstream/sandbox/sandbox.py b/src/buildstream/sandbox/sandbox.py
index ece15c949..8ff0aafa2 100644
--- a/src/buildstream/sandbox/sandbox.py
+++ b/src/buildstream/sandbox/sandbox.py
@@ -410,6 +410,17 @@ class Sandbox():
 
         return 'BST_CAS_DIRECTORIES' in os.environ
 
+    # _fetch_missing_blobs()
+    #
+    # Fetch required file blobs missing from the local cache for sandboxes using
+    # remote execution. This is a no-op for local sandboxes.
+    #
+    # Args:
+    #    vdir (Directory): The virtual directory whose blobs to fetch
+    #
+    def _fetch_missing_blobs(self, vdir):
+        pass
+
     ################################################
     #               Private methods                #
     ################################################