Handle subproject fetching in the Stream class

4 files changed, 23 insertions, 8 deletions

diff --git a/src/buildstream/_frontend/app.py b/src/buildstream/_frontend/app.py
index b55c743e2..68cf7ec4d 100644
--- a/src/buildstream/_frontend/app.py
+++ b/src/buildstream/_frontend/app.py
@@ -243,7 +243,8 @@ class App():
         #
         try:
             self.project = Project(directory, self.context, cli_options=self._main_options['option'],
-                                   default_mirror=self._main_options.get('default_mirror'))
+                                   default_mirror=self._main_options.get('default_mirror'),
+                                   fetch_subprojects=self.stream.fetch_subprojects)
 
             self.stream.set_project(self.project)
         except LoadError as e:
diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py
index 217debf32..fa3539b22 100644
--- a/src/buildstream/_loader/loader.py
+++ b/src/buildstream/_loader/loader.py
@@ -45,11 +45,12 @@ from .._message import Message, MessageType
 # Args:
 #    context (Context): The Context object
 #    project (Project): The toplevel Project object
+#    fetch_subprojects (callable): A function to fetch subprojects
 #    parent (Loader): A parent Loader object, in the case this is a junctioned Loader
 #
 class Loader():
 
-    def __init__(self, context, project, *, parent=None):
+    def __init__(self, context, project, *, fetch_subprojects, parent=None):
 
         # Ensure we have an absolute path for the base directory
         basedir = project.element_path
@@ -69,6 +70,7 @@ class Loader():
         self._basedir = basedir              # Base project directory
         self._first_pass_options = project.first_pass_config.options  # Project options (OptionPool)
         self._parent = parent                # The parent loader
+        self._fetch_subprojects = fetch_subprojects
 
         self._meta_elements = {}  # Dict of resolved meta elements by name
         self._elements = {}       # Dict of elements
@@ -623,7 +625,7 @@ class Loader():
         if element._get_consistency() == Consistency.RESOLVED:
             if ticker:
                 ticker(filename, 'Fetching subproject')
-            element._fetch()
+            self._fetch_subprojects([element])
 
         # Handle the case where a subproject has no ref
         #
@@ -655,7 +657,8 @@ class Loader():
         try:
             from .._project import Project  # pylint: disable=cyclic-import
             project = Project(project_dir, self._context, junction=element,
-                              parent_loader=self, search_for_project=False)
+                              parent_loader=self, search_for_project=False,
+                              fetch_subprojects=self._fetch_subprojects)
         except LoadError as e:
             if e.reason == LoadErrorReason.MISSING_PROJECT_CONF:
                 message = (
diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py
index 0787a7bf4..114d25054 100644
--- a/src/buildstream/_project.py
+++ b/src/buildstream/_project.py
@@ -95,7 +95,7 @@ class Project():
 
     def __init__(self, directory, context, *, junction=None, cli_options=None,
                  default_mirror=None, parent_loader=None,
-                 search_for_project=True):
+                 search_for_project=True, fetch_subprojects=None):
 
         # The project name
         self.name = None
@@ -157,7 +157,7 @@ class Project():
         self._project_includes = None
 
         with PROFILER.profile(Topics.LOAD_PROJECT, self.directory.replace(os.sep, '-')):
-            self._load(parent_loader=parent_loader)
+            self._load(parent_loader=parent_loader, fetch_subprojects=fetch_subprojects)
 
         self._partially_loaded = True
 
@@ -553,7 +553,7 @@ class Project():
     #
     # Raises: LoadError if there was a problem with the project.conf
     #
-    def _load(self, parent_loader=None):
+    def _load(self, *, parent_loader=None, fetch_subprojects):
 
         # Load builtin default
         projectfile = os.path.join(self.directory, _PROJECT_CONF_FILE)
@@ -608,7 +608,7 @@ class Project():
         self._fatal_warnings = _yaml.node_get(pre_config_node, list, 'fatal-warnings', default_value=[])
 
         self.loader = Loader(self._context, self,
-                             parent=parent_loader)
+                             parent=parent_loader, fetch_subprojects=fetch_subprojects)
 
         self._project_includes = Includes(self.loader, copy_tree=False)
 
diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py
index c9e847319..650202d21 100644
--- a/src/buildstream/_stream.py
+++ b/src/buildstream/_stream.py
@@ -953,6 +953,17 @@ class Stream():
 
         return list(output_elements)
 
+    # fetch_subprojects()
+    #
+    # Fetch subprojects as part of the project and element loading process.
+    #
+    # Args:
+    #    junctions (list of Element): The junctions to fetch
+    #
+    def fetch_subprojects(self, junctions):
+        for junction in junctions:
+            junction._fetch()
+
     #############################################################
     #                 Scheduler API forwarding                  #
     #############################################################