element.py: Optimize assemble_done()juerg/assemble_done

After a successful build we know that the artifact is cached. Avoid
querying buildbox-casd and the filesystem.

3 files changed, 24 insertions, 9 deletions

diff --git a/src/buildstream/_artifact.py b/src/buildstream/_artifact.py
index feba3898b..ae1b395b3 100644
--- a/src/buildstream/_artifact.py
+++ b/src/buildstream/_artifact.py
@@ -459,6 +459,14 @@ class Artifact:
     def reset_cached(self):
         self._cached = None
 
+    # set_cached()
+    #
+    # Mark the artifact as cached without querying the filesystem.
+    # This is used as optimization when we know the artifact is available.
+    #
+    def set_cached(self):
+        self._cached = True
+
     # _get_proto()
     #
     # Returns:
diff --git a/src/buildstream/_scheduler/queues/buildqueue.py b/src/buildstream/_scheduler/queues/buildqueue.py
index d98b49476..5cbd5af57 100644
--- a/src/buildstream/_scheduler/queues/buildqueue.py
+++ b/src/buildstream/_scheduler/queues/buildqueue.py
@@ -23,6 +23,7 @@ from datetime import timedelta
 from . import Queue, QueueStatus
 from ..resources import ResourceType
 from ..._message import MessageType
+from ..jobs import JobStatus
 
 
 # A queue which assembles elements
@@ -80,7 +81,7 @@ class BuildQueue(Queue):
     def done(self, job, element, result, status):
 
         # Inform element in main process that assembly is done
-        element._assemble_done()
+        element._assemble_done(status is JobStatus.OK)
 
     def register_pending_element(self, element):
         # Set a "buildable" callback for an element not yet ready
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index d7ede31f1..d39135b53 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -1546,21 +1546,27 @@ class Element(Plugin):
 
     # _assemble_done():
     #
-    # This is called in the main process after the element has been assembled
-    # and in the a subprocess after assembly completes.
+    # This is called in the main process after the element has been assembled.
     #
     # This will result in updating the element state.
     #
-    def _assemble_done(self):
+    # Args:
+    #     successful (bool): Whether the build was successful
+    #
+    def _assemble_done(self, successful):
         assert self.__assemble_scheduled
 
         self.__assemble_scheduled = False
         self.__assemble_done = True
 
-        # Artifact may have a cached success now.
-        if self.__strict_artifact:
-            self.__strict_artifact.reset_cached()
-        if self.__artifact:
+        self.__strict_artifact.reset_cached()
+
+        if successful:
+            # Directly set known cached status as optimization to avoid
+            # querying buildbox-casd and the filesystem.
+            self.__artifact.set_cached()
+            self.__cached_successfully = True
+        else:
             self.__artifact.reset_cached()
 
         # When we're building in non-strict mode, we may have
@@ -1715,7 +1721,7 @@ class Element(Plugin):
                 pass
 
         # ensure we have cache keys
-        self._assemble_done()
+        self.__update_cache_key_non_strict()
 
         with self.timed_activity("Caching artifact"):
             artifact_size = self.__artifact.cache(rootdir, sandbox_build_dir, collectvdir, buildresult, publicdata)