Do not pull/fetch/build elements that are not requiredjuerg/no-remote-summaries

For `bst build --deps plan`, do not process elements in pull/fetch/build
queues until they are requested by a reverse dependency.

5 files changed, 62 insertions, 4 deletions

diff --git a/buildstream/_scheduler/buildqueue.py b/buildstream/_scheduler/buildqueue.py
index 46ce72ca7..24a124b32 100644
--- a/buildstream/_scheduler/buildqueue.py
+++ b/buildstream/_scheduler/buildqueue.py
@@ -38,6 +38,11 @@ class BuildQueue(Queue):
         # state of dependencies may have changed, recalculate element state
         element._update_state()
 
+        if not element._is_required():
+            # Artifact is not currently required but it may be requested later.
+            # Keep it in the queue.
+            return QueueStatus.WAIT
+
         if element._cached():
             return QueueStatus.SKIP
 
diff --git a/buildstream/_scheduler/fetchqueue.py b/buildstream/_scheduler/fetchqueue.py
index c2bceb270..61055725d 100644
--- a/buildstream/_scheduler/fetchqueue.py
+++ b/buildstream/_scheduler/fetchqueue.py
@@ -47,6 +47,11 @@ class FetchQueue(Queue):
         # state of dependencies may have changed, recalculate element state
         element._update_state()
 
+        if not element._is_required():
+            # Artifact is not currently required but it may be requested later.
+            # Keep it in the queue.
+            return QueueStatus.WAIT
+
         # Optionally skip elements that are already in the artifact cache
         if self._skip_cached:
             if not element._can_query_cache():
diff --git a/buildstream/_scheduler/pullqueue.py b/buildstream/_scheduler/pullqueue.py
index 0d6b5dbfb..f9928a342 100644
--- a/buildstream/_scheduler/pullqueue.py
+++ b/buildstream/_scheduler/pullqueue.py
@@ -39,6 +39,11 @@ class PullQueue(Queue):
         # state of dependencies may have changed, recalculate element state
         element._update_state()
 
+        if not element._is_required():
+            # Artifact is not currently required but it may be requested later.
+            # Keep it in the queue.
+            return QueueStatus.WAIT
+
         if not element._can_query_cache():
             return QueueStatus.WAIT
 
diff --git a/buildstream/_stream.py b/buildstream/_stream.py
index 970735898..7ff75cad2 100644
--- a/buildstream/_stream.py
+++ b/buildstream/_stream.py
@@ -181,7 +181,8 @@ class Stream():
                        track_except_targets=track_except,
                        track_cross_junctions=track_cross_junctions,
                        use_artifact_config=True,
-                       fetch_subprojects=True)
+                       fetch_subprojects=True,
+                       dynamic_plan=True)
 
         # Remove the tracking elements from the main targets
         elements = self._pipeline.subtract_elements(elements, track_elements)
@@ -748,7 +749,8 @@ class Stream():
               track_cross_junctions=False,
               use_artifact_config=False,
               artifact_remote_url=None,
-              fetch_subprojects=False):
+              fetch_subprojects=False,
+              dynamic_plan=False):
 
         # Load rewritable if we have any tracking selection to make
         rewritable = False
@@ -799,6 +801,16 @@ class Stream():
                                                   selected,
                                                   except_elements)
 
+        if selection == PipelineSelection.PLAN and dynamic_plan:
+            # We use a dynamic build plan, only request artifacts of top-level targets,
+            # others are requested dynamically as needed.
+            # This avoids pulling, fetching, or building unneeded build-only dependencies.
+            for element in elements:
+                element._set_required()
+        else:
+            for element in selected:
+                element._set_required()
+
         return selected, track_selected
 
     # _message()
diff --git a/buildstream/element.py b/buildstream/element.py
index d0729084c..6dee27911 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -225,6 +225,7 @@ class Element(Plugin):
         self.__whitelist_regex = None           # Resolved regex object to check if file is allowed to overlap
         self.__staged_sources_directory = None  # Location where Element.stage_sources() was called
         self.__tainted = None                   # Whether the artifact is tainted and should not be shared
+        self.__required = False                 # Whether the artifact is required in the current session
 
         # hash tables of loaded artifact metadata, hashed by key
         self.__metadata_keys = {}                     # Strong and weak keys for this key
@@ -1069,7 +1070,7 @@ class Element(Plugin):
             # until the full cache query below.
             cached = self.__artifacts.contains(self, self.__weak_cache_key)
             if (not self.__assemble_scheduled and not self.__assemble_done and
-                    not cached and not self._pull_pending()):
+                    not cached and not self._pull_pending() and self._is_required()):
                 self._schedule_assemble()
                 return
 
@@ -1091,7 +1092,7 @@ class Element(Plugin):
             self.__strong_cached = self.__artifacts.contains(self, self.__strict_cache_key)
 
         if (not self.__assemble_scheduled and not self.__assemble_done and
-                not self.__cached and not self._pull_pending()):
+                not self.__cached and not self._pull_pending() and self._is_required()):
             # Workspaced sources are considered unstable if a build is pending
             # as the build will modify the contents of the workspace.
             # Determine as early as possible if a build is pending to discard
@@ -1322,15 +1323,45 @@ class Element(Plugin):
         # Ensure deterministic owners of sources at build time
         utils._set_deterministic_user(directory)
 
+    # _set_required():
+    #
+    # Mark this element and its runtime dependencies as required.
+    # This unblocks pull/fetch/build.
+    #
+    def _set_required(self):
+        if self.__required:
+            # Already done
+            return
+
+        self.__required = True
+
+        # Request artifacts of runtime dependencies
+        for dep in self.dependencies(Scope.RUN, recurse=False):
+            dep._set_required()
+
+        self._update_state()
+
+    # _is_required():
+    #
+    # Returns whether this element has been marked as required.
+    #
+    def _is_required(self):
+        return self.__required
+
     # _schedule_assemble():
     #
     # This is called in the main process before the element is assembled
     # in a subprocess.
     #
     def _schedule_assemble(self):
+        assert self._is_required()
         assert not self.__assemble_scheduled
         self.__assemble_scheduled = True
 
+        # Requests artifacts of build dependencies
+        for dep in self.dependencies(Scope.BUILD, recurse=False):
+            dep._set_required()
+
         # Invalidate workspace key as the build modifies the workspace directory
         workspace = self._get_workspace()
         if workspace: