WIP: element.py: Optimize fetch_done() and assemble_done()juerg/cached

3 files changed, 22 insertions, 15 deletions

diff --git a/src/buildstream/_scheduler/queues/buildqueue.py b/src/buildstream/_scheduler/queues/buildqueue.py
index d98b49476..5cbd5af57 100644
--- a/src/buildstream/_scheduler/queues/buildqueue.py
+++ b/src/buildstream/_scheduler/queues/buildqueue.py
@@ -23,6 +23,7 @@ from datetime import timedelta
 from . import Queue, QueueStatus
 from ..resources import ResourceType
 from ..._message import MessageType
+from ..jobs import JobStatus
 
 
 # A queue which assembles elements
@@ -80,7 +81,7 @@ class BuildQueue(Queue):
     def done(self, job, element, result, status):
 
         # Inform element in main process that assembly is done
-        element._assemble_done()
+        element._assemble_done(status is JobStatus.OK)
 
     def register_pending_element(self, element):
         # Set a "buildable" callback for an element not yet ready
diff --git a/src/buildstream/_scheduler/queues/fetchqueue.py b/src/buildstream/_scheduler/queues/fetchqueue.py
index 4f38f377a..bc2960fbb 100644
--- a/src/buildstream/_scheduler/queues/fetchqueue.py
+++ b/src/buildstream/_scheduler/queues/fetchqueue.py
@@ -69,7 +69,7 @@ class FetchQueue(Queue):
         if status is JobStatus.FAIL:
             return
 
-        element._fetch_done()
+        element._fetch_done(status is JobStatus.OK, fetch_original=self._should_fetch_original)
 
         # Successful fetch, we must be CACHED or in the sourcecache
         if self._should_fetch_original:
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 106460b9a..42ab1472d 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -1560,17 +1560,20 @@ class Element(Plugin):
     #
     # This will result in updating the element state.
     #
-    def _assemble_done(self):
+    def _assemble_done(self, successful):
         assert self.__assemble_scheduled
 
         self.__assemble_scheduled = False
         self.__assemble_done = True
 
-        # Artifact may have a cached success now.
-        if self.__strict_artifact:
-            self.__strict_artifact.reset_cached()
-        if self.__artifact:
-            self.__artifact.reset_cached()
+        if successful:
+            self.__artifact._cached = True
+            self.__cached_successfully = True
+        else:
+            if self.__strict_artifact:
+                self.__strict_artifact.reset_cached()
+            if self.__artifact:
+                self.__artifact.reset_cached()
 
         # When we're building in non-strict mode, we may have
         # assembled everything to this point without a strong cache
@@ -1724,7 +1727,7 @@ class Element(Plugin):
                 pass
 
         # ensure we have cache keys
-        self._assemble_done()
+        self.__update_cache_key_non_strict()
 
         with self.timed_activity("Caching artifact"):
             artifact_size = self.__artifact.cache(rootdir, sandbox_build_dir, collectvdir, buildresult, publicdata)
@@ -1744,15 +1747,18 @@ class Element(Plugin):
     #
     # Indicates that fetching the sources for this element has been done.
     #
-    def _fetch_done(self):
+    def _fetch_done(self, successful, *, fetch_original):
         # We are not updating the state recursively here since fetching can
         # never end up in updating them.
 
-        # Fetching changes the source state from RESOLVED to CACHED
-        # Fetching cannot change the source state from INCONSISTENT to CACHED because
-        # we prevent fetching when it's INCONSISTENT.
-        # Therefore, only the source state will change.
-        self.__update_source_state()
+        if successful and not fetch_original:
+            self.__source_cached = True
+        else:
+            # Fetching changes the source state from RESOLVED to CACHED
+            # Fetching cannot change the source state from INCONSISTENT to CACHED because
+            # we prevent fetching when it's INCONSISTENT.
+            # Therefore, only the source state will change.
+            self.__update_source_state()
 
     # _pull_pending()
     #