Add support for include '(@)' in project.conf and .bst files

Fixes #331.

14 files changed, 550 insertions, 202 deletions

diff --git a/buildstream/_exceptions.py b/buildstream/_exceptions.py
index 5187357c5..3fb5e5775 100644
--- a/buildstream/_exceptions.py
+++ b/buildstream/_exceptions.py
@@ -214,6 +214,9 @@ class LoadErrorReason(Enum):
     # socket)
     PROJ_PATH_INVALID_KIND = 20
 
+    # A recursive include has been encountered.
+    RECURSIVE_INCLUDE = 21
+
 
 # LoadError
 #
diff --git a/buildstream/_frontend/app.py b/buildstream/_frontend/app.py
index 4039716c8..5c13bf0bf 100644
--- a/buildstream/_frontend/app.py
+++ b/buildstream/_frontend/app.py
@@ -67,7 +67,6 @@ class App():
         self.context = None        # The Context object
         self.stream = None         # The Stream object
         self.project = None        # The toplevel Project object
-        self.loader = None
         self.logger = None         # The LogLine object
         self.interactive = None    # Whether we are running in interactive mode
         self.colors = None         # Whether to use colors in logging
@@ -220,7 +219,6 @@ class App():
         try:
             self.project = Project(directory, self.context, cli_options=self._main_options['option'],
                                    default_mirror=self._main_options.get('default_mirror'))
-            self.loader = self.project.loader
         except LoadError as e:
 
             # Let's automatically start a `bst init` session in this case
@@ -241,7 +239,6 @@ class App():
 
         # Create the stream right away, we'll need to pass it around
         self.stream = Stream(self.context, self.project, self._session_start,
-                             self.loader,
                              session_start_callback=self.session_start_cb,
                              interrupt_callback=self._interrupt_handler,
                              ticker_callback=self._tick,
diff --git a/buildstream/_frontend/widget.py b/buildstream/_frontend/widget.py
index 9e8534f7a..3abc31d40 100644
--- a/buildstream/_frontend/widget.py
+++ b/buildstream/_frontend/widget.py
@@ -483,8 +483,11 @@ class LogLine(Widget):
             text += '\n'
 
         # Plugins
-        text += self._format_plugins(project._element_factory.loaded_dependencies,
-                                     project._source_factory.loaded_dependencies)
+        text += self._format_plugins(project.first_pass_config.element_factory.loaded_dependencies,
+                                     project.first_pass_config.source_factory.loaded_dependencies)
+        if project.config.element_factory and project.config.source_factory:
+            text += self._format_plugins(project.config.element_factory.loaded_dependencies,
+                                         project.config.source_factory.loaded_dependencies)
 
         # Pipeline state
         text += self.content_profile.fmt("Pipeline\n", bold=True)
diff --git a/buildstream/_includes.py b/buildstream/_includes.py
new file mode 100644
index 000000000..e4afeaf82
--- /dev/null
+++ b/buildstream/_includes.py
@@ -0,0 +1,128 @@
+import os
+from collections import Mapping
+from . import _yaml
+from ._exceptions import LoadError, LoadErrorReason
+
+
+# Includes()
+#
+# This takes care of processing include directives "(@)".
+#
+# Args:
+#    loader (Loader): The Loader object
+class Includes:
+
+    def __init__(self, loader):
+        self._loader = loader
+        self._loaded = {}
+
+    # process()
+    #
+    # Process recursively include directives in a YAML node.
+    #
+    # Args:
+    #    node (dict): A YAML node
+    #    included (set): Fail for recursion if trying to load any files in this set
+    #    current_loader (Loader): Use alternative loader (for junction files)
+    #    only_local (bool): Whether to ignore junction files
+    def process(self, node, *,
+                included=set(),
+                current_loader=None,
+                only_local=False):
+        if current_loader is None:
+            current_loader = self._loader
+
+        if isinstance(node.get('(@)'), str):
+            includes = [_yaml.node_get(node, str, '(@)')]
+        else:
+            includes = _yaml.node_get(node, list, '(@)', default_value=None)
+        if '(@)' in node:
+            del node['(@)']
+
+        if includes:
+            for include in reversed(includes):
+                if only_local and ':' in include:
+                    continue
+                include_node, file_path, sub_loader = self._include_file(include,
+                                                                         current_loader)
+                if file_path in included:
+                    provenance = _yaml.node_get_provenance(node)
+                    raise LoadError(LoadErrorReason.RECURSIVE_INCLUDE,
+                                    "{}: trying to recursively include {}". format(provenance,
+                                                                                   file_path))
+                # Because the included node will be modified, we need
+                # to copy it so that we do not modify the toplevel
+                # node of the provenance.
+                include_node = _yaml.node_chain_copy(include_node)
+
+                try:
+                    included.add(file_path)
+                    self.process(include_node, included=included,
+                                 current_loader=sub_loader,
+                                 only_local=only_local)
+                finally:
+                    included.remove(file_path)
+
+                _yaml.composite(include_node, node)
+                to_delete = [key for key, _ in _yaml.node_items(node) if key not in include_node]
+                for key, value in include_node.items():
+                    node[key] = value
+                for key in to_delete:
+                    del node[key]
+
+        for _, value in _yaml.node_items(node):
+            self._process_value(value,
+                                included=included,
+                                current_loader=current_loader,
+                                only_local=only_local)
+
+    # _include_file()
+    #
+    # Load include YAML file from with a loader.
+    #
+    # Args:
+    #    include (str): file path relative to loader's project directory.
+    #                   Can be prefixed with junctio name.
+    #    loader (Loader): Loader for the current project.
+    def _include_file(self, include, loader):
+        shortname = include
+        if ':' in include:
+            junction, include = include.split(':', 1)
+            junction_loader = loader._get_loader(junction, fetch_subprojects=True)
+            current_loader = junction_loader
+        else:
+            current_loader = loader
+        project = current_loader.project
+        directory = project.directory
+        file_path = os.path.join(directory, include)
+        key = (current_loader, file_path)
+        if file_path not in self._loaded:
+            self._loaded[key] = _yaml.load(os.path.join(directory, include),
+                                           shortname=shortname,
+                                           project=project)
+        return self._loaded[key], file_path, current_loader
+
+    # _process_value()
+    #
+    # Select processing for value that could be a list or a dictionary.
+    #
+    # Args:
+    #    value: Value to process. Can be a list or a dictionary.
+    #    included (set): Fail for recursion if trying to load any files in this set
+    #    current_loader (Loader): Use alternative loader (for junction files)
+    #    only_local (bool): Whether to ignore junction files
+    def _process_value(self, value, *,
+                       included=set(),
+                       current_loader=None,
+                       only_local=False):
+        if isinstance(value, Mapping):
+            self.process(value,
+                         included=included,
+                         current_loader=current_loader,
+                         only_local=only_local)
+        elif isinstance(value, list):
+            for v in value:
+                self._process_value(v,
+                                    included=included,
+                                    current_loader=current_loader,
+                                    only_local=only_local)
diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py
index 280805981..6e46197ab 100644
--- a/buildstream/_loader/loader.py
+++ b/buildstream/_loader/loader.py
@@ -29,6 +29,7 @@ from .. import _yaml
 from ..element import Element
 from .._profile import Topics, profile_start, profile_end
 from .._platform import Platform
+from .._includes import Includes
 
 from .types import Symbol, Dependency
 from .loadelement import LoadElement
@@ -69,6 +70,7 @@ class Loader():
         self._context = context
         self._options = project.options      # Project options (OptionPool)
         self._basedir = basedir              # Base project directory
+        self._first_pass_options = project.first_pass_config.options  # Project options (OptionPool)
         self._tempdir = tempdir              # A directory to cleanup
         self._parent = parent                # The parent loader
 
@@ -76,6 +78,8 @@ class Loader():
         self._elements = {}       # Dict of elements
         self._loaders = {}        # Dict of junction loaders
 
+        self._includes = Includes(self)
+
     # load():
     #
     # Loads the project based on the parameters given to the constructor
@@ -215,7 +219,7 @@ class Loader():
         # Load the data and process any conditional statements therein
         fullpath = os.path.join(self._basedir, filename)
         try:
-            node = _yaml.load(fullpath, shortname=filename, copy_tree=rewritable)
+            node = _yaml.load(fullpath, shortname=filename, copy_tree=rewritable, project=self.project)
         except LoadError as e:
             if e.reason == LoadErrorReason.MISSING_FILE:
                 # If we can't find the file, try to suggest plausible
@@ -241,7 +245,15 @@ class Loader():
                                 message, detail=detail) from e
             else:
                 raise
-        self._options.process_node(node)
+        kind = _yaml.node_get(node, str, Symbol.KIND)
+        if kind == "junction":
+            self._first_pass_options.process_node(node)
+        else:
+            self.project.ensure_fully_loaded()
+
+            self._includes.process(node)
+
+            self._options.process_node(node)
 
         element = LoadElement(node, filename, self)
 
@@ -433,7 +445,8 @@ class Loader():
                                    _yaml.node_get(node, Mapping, Symbol.ENVIRONMENT, default_value={}),
                                    _yaml.node_get(node, list, Symbol.ENV_NOCACHE, default_value=[]),
                                    _yaml.node_get(node, Mapping, Symbol.PUBLIC, default_value={}),
-                                   _yaml.node_get(node, Mapping, Symbol.SANDBOX, default_value={}))
+                                   _yaml.node_get(node, Mapping, Symbol.SANDBOX, default_value={}),
+                                   element_kind == 'junction')
 
         # Cache it now, make sure it's already there before recursing
         self._meta_elements[element_name] = meta_element
diff --git a/buildstream/_loader/metaelement.py b/buildstream/_loader/metaelement.py
index 16788e92b..c13d5591e 100644
--- a/buildstream/_loader/metaelement.py
+++ b/buildstream/_loader/metaelement.py
@@ -36,9 +36,11 @@ class MetaElement():
     #    env_nocache: List of environment vars which should not be considered in cache keys
     #    public: Public domain data dictionary
     #    sandbox: Configuration specific to the sandbox environment
+    #    first_pass: The element is to be loaded with first pass configuration (junction)
     #
     def __init__(self, project, name, kind, provenance, sources, config,
-                 variables, environment, env_nocache, public, sandbox):
+                 variables, environment, env_nocache, public, sandbox,
+                 first_pass):
         self.project = project
         self.name = name
         self.kind = kind
@@ -52,3 +54,4 @@ class MetaElement():
         self.sandbox = sandbox
         self.build_dependencies = []
         self.dependencies = []
+        self.first_pass = first_pass
diff --git a/buildstream/_loader/metasource.py b/buildstream/_loader/metasource.py
index 3bcc21ec6..da2c0e292 100644
--- a/buildstream/_loader/metasource.py
+++ b/buildstream/_loader/metasource.py
@@ -30,6 +30,7 @@ class MetaSource():
     #    element_kind: The kind of the owning element
     #    kind: The kind of the source
     #    config: The configuration data for the source
+    #    first_pass: This source will be used with first project pass configuration (used for junctions).
     #
     def __init__(self, element_name, element_index, element_kind, kind, config, directory):
         self.element_name = element_name
@@ -38,3 +39,4 @@ class MetaSource():
         self.kind = kind
         self.config = config
         self.directory = directory
+        self.first_pass = False
diff --git a/buildstream/_options/optionpool.py b/buildstream/_options/optionpool.py
index f90fd820c..b53e87a3d 100644
--- a/buildstream/_options/optionpool.py
+++ b/buildstream/_options/optionpool.py
@@ -107,16 +107,19 @@ class OptionPool():
     #
     # Args:
     #    cli_options (list): A list of (str, str) tuples
+    #    ignore_unknown (bool): Whether to silently ignore unknown options.
     #
-    def load_cli_values(self, cli_options):
+    def load_cli_values(self, cli_options, *, ignore_unknown=False):
         for option_name, option_value in cli_options:
             try:
                 option = self._options[option_name]
             except KeyError as e:
-                raise LoadError(LoadErrorReason.INVALID_DATA,
-                                "Unknown option '{}' specified on the command line"
-                                .format(option_name)) from e
-            option.set_value(option_value)
+                if not ignore_unknown:
+                    raise LoadError(LoadErrorReason.INVALID_DATA,
+                                    "Unknown option '{}' specified on the command line"
+                                    .format(option_name)) from e
+            else:
+                option.set_value(option_value)
 
     # resolve()
     #
diff --git a/buildstream/_project.py b/buildstream/_project.py
index faecec87f..a0f3a5059 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -37,6 +37,7 @@ from ._versions import BST_FORMAT_VERSION
 from ._loader import Loader
 from .element import Element
 from ._message import Message, MessageType
+from ._includes import Includes
 
 
 # Project Configuration file
@@ -65,6 +66,20 @@ class HostMount():
             self.host_path = self.path
 
 
+# Represents project configuration that can have different values for junctions.
+class ProjectConfig:
+    def __init__(self):
+        self.element_factory = None
+        self.source_factory = None
+        self.options = None                      # OptionPool
+        self.base_variables = {}                 # The base set of variables
+        self.element_overrides = {}              # Element specific configurations
+        self.source_overrides = {}               # Source specific configurations
+        self.mirrors = OrderedDict()             # contains dicts of alias-mappings to URIs.
+        self.default_mirror = None               # The name of the preferred mirror.
+        self._aliases = {}                       # Aliases dictionary
+
+
 # Project()
 #
 # The Project Configuration
@@ -87,23 +102,21 @@ class Project():
         self.refs = ProjectRefs(self.directory, 'project.refs')
         self.junction_refs = ProjectRefs(self.directory, 'junction.refs')
 
-        self.options = None                      # OptionPool
+        self.config = ProjectConfig()
+        self.first_pass_config = ProjectConfig()
+
         self.junction = junction                 # The junction Element object, if this is a subproject
         self.fail_on_overlap = False             # Whether overlaps are treated as errors
         self.ref_storage = None                  # ProjectRefStorage setting
-        self.base_variables = {}                 # The base set of variables
         self.base_environment = {}               # The base set of environment variables
         self.base_env_nocache = None             # The base nocache mask (list) for the environment
-        self.element_overrides = {}              # Element specific configurations
-        self.source_overrides = {}               # Source specific configurations
-        self.mirrors = OrderedDict()             # contains dicts of alias-mappings to URIs.
-        self.default_mirror = default_mirror     # The name of the preferred mirror.
 
         #
         # Private Members
         #
         self._context = context  # The invocation Context
-        self._aliases = {}       # Aliases dictionary
+
+        self._default_mirror = default_mirror    # The name of the preferred mirror.
 
         self._cli_options = cli_options
         self._cache_key = None
@@ -112,18 +125,37 @@ class Project():
         self._shell_environment = {}  # Statically set environment vars
         self._shell_host_files = []   # A list of HostMount objects
 
-        self._element_factory = None
-        self._source_factory = None
+        self.artifact_cache_specs = None
+        self._sandbox = None
+        self._splits = None
+
+        self._context.add_project(self)
+
+        self._partially_loaded = False
+        self._fully_loaded = False
+        self._project_includes = None
 
         profile_start(Topics.LOAD_PROJECT, self.directory.replace(os.sep, '-'))
-        self._load()
+        self._load(parent_loader=parent_loader, tempdir=tempdir)
         profile_end(Topics.LOAD_PROJECT, self.directory.replace(os.sep, '-'))
 
-        self._context.add_project(self)
+        self._partially_loaded = True
 
-        self.loader = Loader(self._context, self,
-                             parent=parent_loader,
-                             tempdir=tempdir)
+    @property
+    def options(self):
+        return self.config.options
+
+    @property
+    def base_variables(self):
+        return self.config.base_variables
+
+    @property
+    def element_overrides(self):
+        return self.config.element_overrides
+
+    @property
+    def source_overrides(self):
+        return self.config.source_overrides
 
     # translate_url():
     #
@@ -132,6 +164,7 @@ class Project():
     #
     # Args:
     #    url (str): A url, which may be using an alias
+    #    first_pass (bool): Whether to use first pass configuration (for junctions)
     #
     # Returns:
     #    str: The fully qualified url, with aliases resolved
@@ -139,10 +172,15 @@ class Project():
     # This method is provided for :class:`.Source` objects to resolve
     # fully qualified urls based on the shorthand which is allowed
     # to be specified in the YAML
-    def translate_url(self, url):
+    def translate_url(self, url, *, first_pass=False):
+        if first_pass:
+            config = self.first_pass_config
+        else:
+            config = self.config
+
         if url and utils._ALIAS_SEPARATOR in url:
             url_alias, url_body = url.split(utils._ALIAS_SEPARATOR, 1)
-            alias_url = self._aliases.get(url_alias)
+            alias_url = config._aliases.get(url_alias)
             if alias_url:
                 url = alias_url + url_body
 
@@ -183,12 +221,16 @@ class Project():
     # Args:
     #    artifacts (ArtifactCache): The artifact cache
     #    meta (MetaElement): The loaded MetaElement
+    #    first_pass (bool): Whether to use first pass configuration (for junctions)
     #
     # Returns:
     #    (Element): A newly created Element object of the appropriate kind
     #
-    def create_element(self, artifacts, meta):
-        return self._element_factory.create(self._context, self, artifacts, meta)
+    def create_element(self, artifacts, meta, *, first_pass=False):
+        if first_pass:
+            return self.first_pass_config.element_factory.create(self._context, self, artifacts, meta)
+        else:
+            return self.config.element_factory.create(self._context, self, artifacts, meta)
 
     # create_source()
     #
@@ -196,12 +238,16 @@ class Project():
     #
     # Args:
     #    meta (MetaSource): The loaded MetaSource
+    #    first_pass (bool): Whether to use first pass configuration (for junctions)
     #
     # Returns:
     #    (Source): A newly created Source object of the appropriate kind
     #
-    def create_source(self, meta):
-        return self._source_factory.create(self._context, self, meta)
+    def create_source(self, meta, *, first_pass=False):
+        if first_pass:
+            return self.first_pass_config.source_factory.create(self._context, self, meta)
+        else:
+            return self.config.source_factory.create(self._context, self, meta)
 
     # get_alias_uri()
     #
@@ -209,28 +255,43 @@ class Project():
     #
     # Args:
     #    alias (str): The alias.
+    #    first_pass (bool): Whether to use first pass configuration (for junctions)
     #
     # Returns:
     #    str: The URI for the given alias; or None: if there is no URI for
     #         that alias.
-    def get_alias_uri(self, alias):
-        return self._aliases.get(alias)
+    def get_alias_uri(self, alias, *, first_pass=False):
+        if first_pass:
+            config = self.first_pass_config
+        else:
+            config = self.config
+
+        return config._aliases.get(alias)
 
     # get_alias_uris()
     #
+    # Args:
+    #    alias (str): The alias.
+    #    first_pass (bool): Whether to use first pass configuration (for junctions)
+    #
     # Returns a list of every URI to replace an alias with
-    def get_alias_uris(self, alias):
-        if not alias or alias not in self._aliases:
+    def get_alias_uris(self, alias, *, first_pass=False):
+        if first_pass:
+            config = self.first_pass_config
+        else:
+            config = self.config
+
+        if not alias or alias not in config._aliases:
             return [None]
 
         mirror_list = []
-        for key, alias_mapping in self.mirrors.items():
+        for key, alias_mapping in config.mirrors.items():
             if alias in alias_mapping:
-                if key == self.default_mirror:
+                if key == config.default_mirror:
                     mirror_list = alias_mapping[alias] + mirror_list
                 else:
                     mirror_list += alias_mapping[alias]
-        mirror_list.append(self._aliases[alias])
+        mirror_list.append(config._aliases[alias])
         return mirror_list
 
     # load_elements()
@@ -276,6 +337,23 @@ class Project():
 
         return elements
 
+    # ensure_fully_loaded()
+    #
+    # Ensure project has finished loading. At first initialization, a
+    # project can only load junction elements. Other elements require
+    # project to be fully loaded.
+    #
+    def ensure_fully_loaded(self):
+        if self._fully_loaded:
+            return
+        assert self._partially_loaded
+        self._fully_loaded = True
+
+        if self.junction:
+            self.junction._get_project().ensure_fully_loaded()
+
+        self._load_second_pass()
+
     # cleanup()
     #
     # Cleans up resources used loading elements
@@ -288,36 +366,29 @@ class Project():
 
     # _load():
     #
-    # Loads the project configuration file in the project directory.
+    # Loads the project configuration file in the project
+    # directory process the first pass.
     #
     # Raises: LoadError if there was a problem with the project.conf
     #
-    def _load(self):
+    def _load(self, parent_loader=None, tempdir=None):
 
         # Load builtin default
         projectfile = os.path.join(self.directory, _PROJECT_CONF_FILE)
-        config = _yaml.load(_site.default_project_config)
+        self._default_config_node = _yaml.load(_site.default_project_config)
 
         # Load project local config and override the builtin
         try:
-            project_conf = _yaml.load(projectfile)
+            self._project_conf = _yaml.load(projectfile)
         except LoadError as e:
             # Raise a more specific error here
             raise LoadError(LoadErrorReason.MISSING_PROJECT_CONF, str(e))
 
-        _yaml.composite(config, project_conf)
-
-        # Element and Source  type configurations will be composited later onto
-        # element/source types, so we delete it from here and run our final
-        # assertion after.
-        self.element_overrides = _yaml.node_get(config, Mapping, 'elements', default_value={})
-        self.source_overrides = _yaml.node_get(config, Mapping, 'sources', default_value={})
-        config.pop('elements', None)
-        config.pop('sources', None)
-        _yaml.node_final_assertions(config)
+        pre_config_node = _yaml.node_copy(self._default_config_node)
+        _yaml.composite(pre_config_node, self._project_conf)
 
         # Assert project's format version early, before validating toplevel keys
-        format_version = _yaml.node_get(config, int, 'format-version')
+        format_version = _yaml.node_get(pre_config_node, int, 'format-version')
         if BST_FORMAT_VERSION < format_version:
             major, minor = utils.get_bst_version()
             raise LoadError(
@@ -325,59 +396,70 @@ class Project():
                 "Project requested format version {}, but BuildStream {}.{} only supports up until format version {}"
                 .format(format_version, major, minor, BST_FORMAT_VERSION))
 
-        _yaml.node_validate(config, [
-            'format-version',
-            'element-path', 'variables',
-            'environment', 'environment-nocache',
-            'split-rules', 'elements', 'plugins',
-            'aliases', 'name',
-            'artifacts', 'options',
-            'fail-on-overlap', 'shell',
-            'ref-storage', 'sandbox', 'mirrors',
-        ])
-
         # The project name, element path and option declarations
         # are constant and cannot be overridden by option conditional statements
-        self.name = _yaml.node_get(config, str, 'name')
+        self.name = _yaml.node_get(pre_config_node, str, 'name')
 
         # Validate that project name is a valid symbol name
-        _yaml.assert_symbol_name(_yaml.node_get_provenance(config, 'name'),
+        _yaml.assert_symbol_name(_yaml.node_get_provenance(pre_config_node, 'name'),
                                  self.name, "project name")
 
         self.element_path = os.path.join(
             self.directory,
-            _yaml.node_get_project_path(config, 'element-path', self.directory,
+            _yaml.node_get_project_path(pre_config_node, 'element-path', self.directory,
                                         check_is_dir=True)
         )
 
-        # Load project options
-        options_node = _yaml.node_get(config, Mapping, 'options', default_value={})
-        self.options = OptionPool(self.element_path)
-        self.options.load(options_node)
-        if self.junction:
-            # load before user configuration
-            self.options.load_yaml_values(self.junction.options, transform=self.junction._subst_string)
+        self.config.options = OptionPool(self.element_path)
+        self.first_pass_config.options = OptionPool(self.element_path)
 
-        # Collect option values specified in the user configuration
-        overrides = self._context.get_overrides(self.name)
-        override_options = _yaml.node_get(overrides, Mapping, 'options', default_value={})
-        self.options.load_yaml_values(override_options)
-        if self._cli_options:
-            self.options.load_cli_values(self._cli_options)
+        self.loader = Loader(self._context, self,
+                             parent=parent_loader,
+                             tempdir=tempdir)
 
-        # We're done modifying options, now we can use them for substitutions
-        self.options.resolve()
+        self._project_includes = Includes(self.loader)
 
-        #
-        # Now resolve any conditionals in the remaining configuration,
-        # any conditionals specified for project option declarations,
-        # or conditionally specifying the project name; will be ignored.
-        #
-        self.options.process_node(config)
+        project_conf_first_pass = _yaml.node_copy(self._project_conf)
+        self._project_includes.process(project_conf_first_pass, only_local=True)
+        config_no_include = _yaml.node_copy(self._default_config_node)
+        _yaml.composite(config_no_include, project_conf_first_pass)
 
-        # Override default_mirror if not set by command-line
-        if not self.default_mirror:
-            self.default_mirror = _yaml.node_get(overrides, str, 'default-mirror', default_value=None)
+        self._load_pass(config_no_include, self.first_pass_config,
+                        ignore_unknown=True)
+
+        # Use separate file for storing source references
+        self.ref_storage = _yaml.node_get(pre_config_node, str, 'ref-storage')
+        if self.ref_storage not in [ProjectRefStorage.INLINE, ProjectRefStorage.PROJECT_REFS]:
+            p = _yaml.node_get_provenance(pre_config_node, 'ref-storage')
+            raise LoadError(LoadErrorReason.INVALID_DATA,
+                            "{}: Invalid value '{}' specified for ref-storage"
+                            .format(p, self.ref_storage))
+
+        if self.ref_storage == ProjectRefStorage.PROJECT_REFS:
+            self.junction_refs.load(self.first_pass_config.options)
+
+    # _load_second_pass()
+    #
+    # Process the second pass of loading the project configuration.
+    #
+    def _load_second_pass(self):
+        project_conf_second_pass = _yaml.node_copy(self._project_conf)
+        self._project_includes.process(project_conf_second_pass)
+        config = _yaml.node_copy(self._default_config_node)
+        _yaml.composite(config, project_conf_second_pass)
+
+        self._load_pass(config, self.config)
+
+        _yaml.node_validate(config, [
+            'format-version',
+            'element-path', 'variables',
+            'environment', 'environment-nocache',
+            'split-rules', 'elements', 'plugins',
+            'aliases', 'name',
+            'artifacts', 'options',
+            'fail-on-overlap', 'shell',
+            'ref-storage', 'sandbox', 'mirrors'
+        ])
 
         #
         # Now all YAML composition is done, from here on we just load
@@ -387,24 +469,6 @@ class Project():
         # Load artifacts pull/push configuration for this project
         self.artifact_cache_specs = ArtifactCache.specs_from_config_node(config, self.directory)
 
-        self._load_plugin_factories(config)
-
-        # Source url aliases
-        self._aliases = _yaml.node_get(config, Mapping, 'aliases', default_value={})
-
-        # Load base variables
-        self.base_variables = _yaml.node_get(config, Mapping, 'variables')
-
-        # Add the project name as a default variable
-        self.base_variables['project-name'] = self.name
-
-        # Extend variables with automatic variables and option exports
-        # Initialize it as a string as all variables are processed as strings.
-        self.base_variables['max-jobs'] = str(multiprocessing.cpu_count())
-
-        # Export options into variables, if that was requested
-        self.options.export_variables(self.base_variables)
-
         # Load sandbox environment variables
         self.base_environment = _yaml.node_get(config, Mapping, 'environment')
         self.base_env_nocache = _yaml.node_get(config, list, 'environment-nocache')
@@ -418,18 +482,9 @@ class Project():
         # Fail on overlap
         self.fail_on_overlap = _yaml.node_get(config, bool, 'fail-on-overlap')
 
-        # Use separate file for storing source references
-        self.ref_storage = _yaml.node_get(config, str, 'ref-storage')
-        if self.ref_storage not in [ProjectRefStorage.INLINE, ProjectRefStorage.PROJECT_REFS]:
-            p = _yaml.node_get_provenance(config, 'ref-storage')
-            raise LoadError(LoadErrorReason.INVALID_DATA,
-                            "{}: Invalid value '{}' specified for ref-storage"
-                            .format(p, self.ref_storage))
-
         # Load project.refs if it exists, this may be ignored.
         if self.ref_storage == ProjectRefStorage.PROJECT_REFS:
             self.refs.load(self.options)
-            self.junction_refs.load(self.options)
 
         # Parse shell options
         shell_options = _yaml.node_get(config, Mapping, 'shell')
@@ -461,6 +516,71 @@ class Project():
 
             self._shell_host_files.append(mount)
 
+    # _load_pass():
+    #
+    # Loads parts of the project configuration that are different
+    # for first and second pass configurations.
+    #
+    # Args:
+    #    config (dict) - YaML node of the configuration file.
+    #    output (ProjectConfig) - ProjectConfig to load configuration onto.
+    #    ignore_unknown (bool) - Whether option loader shoud ignore unknown options.
+    #
+    def _load_pass(self, config, output, *,
+                   ignore_unknown=False):
+
+        # Element and Source  type configurations will be composited later onto
+        # element/source types, so we delete it from here and run our final
+        # assertion after.
+        output.element_overrides = _yaml.node_get(config, Mapping, 'elements', default_value={})
+        output.source_overrides = _yaml.node_get(config, Mapping, 'sources', default_value={})
+        config.pop('elements', None)
+        config.pop('sources', None)
+        _yaml.node_final_assertions(config)
+
+        self._load_plugin_factories(config, output)
+
+        # Load project options
+        options_node = _yaml.node_get(config, Mapping, 'options', default_value={})
+        output.options.load(options_node)
+        if self.junction:
+            # load before user configuration
+            output.options.load_yaml_values(self.junction.options, transform=self.junction._subst_string)
+
+        # Collect option values specified in the user configuration
+        overrides = self._context.get_overrides(self.name)
+        override_options = _yaml.node_get(overrides, Mapping, 'options', default_value={})
+        output.options.load_yaml_values(override_options)
+        if self._cli_options:
+            output.options.load_cli_values(self._cli_options, ignore_unknown=ignore_unknown)
+
+        # We're done modifying options, now we can use them for substitutions
+        output.options.resolve()
+
+        #
+        # Now resolve any conditionals in the remaining configuration,
+        # any conditionals specified for project option declarations,
+        # or conditionally specifying the project name; will be ignored.
+        #
+        output.options.process_node(config)
+
+        # Load base variables
+        output.base_variables = _yaml.node_get(config, Mapping, 'variables')
+
+        # Add the project name as a default variable
+        output.base_variables['project-name'] = self.name
+
+        # Extend variables with automatic variables and option exports
+        # Initialize it as a string as all variables are processed as strings.
+        output.base_variables['max-jobs'] = str(multiprocessing.cpu_count())
+
+        # Export options into variables, if that was requested
+        output.options.export_variables(output.base_variables)
+
+        # Override default_mirror if not set by command-line
+        output.default_mirror = self._default_mirror or _yaml.node_get(overrides, str,
+                                                                       'default-mirror', default_value=None)
+
         mirrors = _yaml.node_get(config, list, 'mirrors', default_value=[])
         for mirror in mirrors:
             allowed_mirror_fields = [
@@ -472,11 +592,38 @@ class Project():
             for alias_mapping, uris in _yaml.node_items(mirror['aliases']):
                 assert isinstance(uris, list)
                 alias_mappings[alias_mapping] = list(uris)
-            self.mirrors[mirror_name] = alias_mappings
-            if not self.default_mirror:
-                self.default_mirror = mirror_name
+            output.mirrors[mirror_name] = alias_mappings
+            if not output.default_mirror:
+                output.default_mirror = mirror_name
 
-    def _load_plugin_factories(self, config):
+        # Source url aliases
+        output._aliases = _yaml.node_get(config, Mapping, 'aliases', default_value={})
+
+    # _ensure_project_dir()
+    #
+    # Returns path of the project directory, if a configuration file is found
+    # in given directory or any of its parent directories.
+    #
+    # Args:
+    #    directory (str) - directory from where the command was invoked
+    #
+    # Raises:
+    #    LoadError if project.conf is not found
+    #
+    def _ensure_project_dir(self, directory):
+        directory = os.path.abspath(directory)
+        while not os.path.isfile(os.path.join(directory, _PROJECT_CONF_FILE)):
+            parent_dir = os.path.dirname(directory)
+            if directory == parent_dir:
+                raise LoadError(
+                    LoadErrorReason.MISSING_PROJECT_CONF,
+                    '{} not found in current directory or any of its parent directories'
+                    .format(_PROJECT_CONF_FILE))
+            directory = parent_dir
+
+        return directory
+
+    def _load_plugin_factories(self, config, output):
         plugin_source_origins = []   # Origins of custom sources
         plugin_element_origins = []  # Origins of custom elements
 
@@ -523,12 +670,12 @@ class Project():
                 self._store_origin(origin, 'elements', plugin_element_origins)
 
         pluginbase = PluginBase(package='buildstream.plugins')
-        self._element_factory = ElementFactory(pluginbase,
-                                               plugin_origins=plugin_element_origins,
-                                               format_versions=element_format_versions)
-        self._source_factory = SourceFactory(pluginbase,
-                                             plugin_origins=plugin_source_origins,
-                                             format_versions=source_format_versions)
+        output.element_factory = ElementFactory(pluginbase,
+                                                plugin_origins=plugin_element_origins,
+                                                format_versions=element_format_versions)
+        output.source_factory = SourceFactory(pluginbase,
+                                              plugin_origins=plugin_source_origins,
+                                              format_versions=source_format_versions)
 
     # _store_origin()
     #
@@ -563,27 +710,3 @@ class Project():
                 # paths are passed in relative to the project, but must be absolute
                 origin_dict['path'] = os.path.join(self.directory, path)
             destination.append(origin_dict)
-
-    # _ensure_project_dir()
-    #
-    # Returns path of the project directory, if a configuration file is found
-    # in given directory or any of its parent directories.
-    #
-    # Args:
-    #    directory (str) - directory from where the command was invoked
-    #
-    # Raises:
-    #    LoadError if project.conf is not found
-    #
-    def _ensure_project_dir(self, directory):
-        directory = os.path.abspath(directory)
-        while not os.path.isfile(os.path.join(directory, _PROJECT_CONF_FILE)):
-            parent_dir = os.path.dirname(directory)
-            if directory == parent_dir:
-                raise LoadError(
-                    LoadErrorReason.MISSING_PROJECT_CONF,
-                    '{} not found in current directory or any of its parent directories'
-                    .format(_PROJECT_CONF_FILE))
-            directory = parent_dir
-
-        return directory
diff --git a/buildstream/_stream.py b/buildstream/_stream.py
index f33d5224c..f17d641de 100644
--- a/buildstream/_stream.py
+++ b/buildstream/_stream.py
@@ -45,7 +45,6 @@ from . import Scope, Consistency
 #    context (Context): The Context object
 #    project (Project): The Project object
 #    session_start (datetime): The time when the session started
-#    loader (Loader): The Loader object
 #    session_start_callback (callable): A callback to invoke when the session starts
 #    interrupt_callback (callable): A callback to invoke when we get interrupted
 #    ticker_callback (callable): Invoked every second while running the scheduler
@@ -54,7 +53,7 @@ from . import Scope, Consistency
 #
 class Stream():
 
-    def __init__(self, context, project, session_start, loader, *,
+    def __init__(self, context, project, session_start, *,
                  session_start_callback=None,
                  interrupt_callback=None,
                  ticker_callback=None,
@@ -906,6 +905,10 @@ class Stream():
         for element in track_selected:
             element._schedule_tracking()
 
+        # ArtifactCache.setup_remotes expects all projects to be fully loaded
+        for project in self._context.get_projects():
+            project.ensure_fully_loaded()
+
         # Connect to remote caches, this needs to be done before resolving element state
         self._artifacts.setup_remotes(use_config=use_artifact_config, remote_url=artifact_remote_url)
 
diff --git a/buildstream/_versions.py b/buildstream/_versions.py
index 4531d9a72..39ff30fc3 100644
--- a/buildstream/_versions.py
+++ b/buildstream/_versions.py
@@ -23,7 +23,7 @@
 # This version is bumped whenever enhancements are made
 # to the `project.conf` format or the core element format.
 #
-BST_FORMAT_VERSION = 11
+BST_FORMAT_VERSION = 12
 
 
 # The base BuildStream artifact version
diff --git a/buildstream/_yaml.py b/buildstream/_yaml.py
index 33ee444aa..66500fbad 100644
--- a/buildstream/_yaml.py
+++ b/buildstream/_yaml.py
@@ -38,6 +38,19 @@ RoundTripConstructor.add_constructor(u'tag:yaml.org,2002:float', RoundTripConstr
 PROVENANCE_KEY = '__bst_provenance_info'
 
 
+# Provides information about file for provenance
+#
+# Args:
+#    name (str): Full path to the file
+#    shortname (str): Relative path to the file
+#    project (Project): Project where the shortname is relative from
+class ProvenanceFile():
+    def __init__(self, name, shortname, project):
+        self.name = name
+        self.shortname = shortname
+        self.project = project
+
+
 # Provenance tracks the origin of a given node in the parsed dictionary.
 #
 # Args:
@@ -57,7 +70,7 @@ class Provenance():
 
     # Convert a Provenance to a string for error reporting
     def __str__(self):
-        return "{} [line {:d} column {:d}]".format(self.filename, self.line, self.col)
+        return "{} [line {:d} column {:d}]".format(self.filename.shortname, self.line, self.col)
 
     # Abstract method
     def clone(self):
@@ -175,13 +188,15 @@ class CompositeTypeError(CompositeError):
 #
 # Raises: LoadError
 #
-def load(filename, shortname=None, copy_tree=False):
+def load(filename, shortname=None, copy_tree=False, *, project=None):
     if not shortname:
         shortname = filename
 
+    file = ProvenanceFile(filename, shortname, project)
+
     try:
         with open(filename) as f:
-            return load_data(f, shortname=shortname, copy_tree=copy_tree)
+            return load_data(f, file, copy_tree=copy_tree)
     except FileNotFoundError as e:
         raise LoadError(LoadErrorReason.MISSING_FILE,
                         "Could not find file at {}".format(filename)) from e
@@ -193,7 +208,7 @@ def load(filename, shortname=None, copy_tree=False):
 
 # Like load(), but doesnt require the data to be in a file
 #
-def load_data(data, shortname=None, copy_tree=False):
+def load_data(data, file=None, copy_tree=False):
 
     try:
         contents = yaml.load(data, yaml.loader.RoundTripLoader, preserve_quotes=True)
@@ -208,9 +223,9 @@ def load_data(data, shortname=None, copy_tree=False):
         else:
             raise LoadError(LoadErrorReason.INVALID_YAML,
                             "YAML file has content of type '{}' instead of expected type 'dict': {}"
-                            .format(type(contents).__name__, shortname))
+                            .format(type(contents).__name__, file.name))
 
-    return node_decorated_copy(shortname, contents, copy_tree=copy_tree)
+    return node_decorated_copy(file, contents, copy_tree=copy_tree)
 
 
 # Dumps a previously loaded YAML node to a file
@@ -506,7 +521,7 @@ def node_items(node):
 def ensure_provenance(node):
     provenance = node.get(PROVENANCE_KEY)
     if not provenance:
-        provenance = DictProvenance('', node, node)
+        provenance = DictProvenance(ProvenanceFile('', '', None), node, node)
     node[PROVENANCE_KEY] = provenance
 
     return provenance
diff --git a/buildstream/element.py b/buildstream/element.py
index bec87815c..e2a032197 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -211,6 +211,11 @@ class Element(Plugin):
 
         super().__init__(meta.name, context, project, meta.provenance, "element")
 
+        self.__is_junction = meta.kind == "junction"
+
+        if not self.__is_junction:
+            project.ensure_fully_loaded()
+
         self.normal_name = os.path.splitext(self.name.replace(os.sep, '-'))[0]
         """A normalized element name
 
@@ -903,16 +908,20 @@ class Element(Plugin):
     @classmethod
     def _new_from_meta(cls, meta, artifacts):
 
+        if not meta.first_pass:
+            meta.project.ensure_fully_loaded()
+
         if meta in cls.__instantiated_elements:
             return cls.__instantiated_elements[meta]
 
-        project = meta.project
-        element = project.create_element(artifacts, meta)
+        element = meta.project.create_element(artifacts, meta, first_pass=meta.first_pass)
         cls.__instantiated_elements[meta] = element
 
         # Instantiate sources
         for meta_source in meta.sources:
-            source = project.create_source(meta_source)
+            meta_source.first_pass = meta.kind == "junction"
+            source = meta.project.create_source(meta_source,
+                                                first_pass=meta.first_pass)
             redundant_ref = source._load_ref()
             element.__sources.append(source)
 
@@ -2166,16 +2175,21 @@ class Element(Plugin):
 
     def __compose_default_splits(self, defaults):
         project = self._get_project()
-        project_splits = _yaml.node_chain_copy(project._splits)
 
         element_public = _yaml.node_get(defaults, Mapping, 'public', default_value={})
         element_bst = _yaml.node_get(element_public, Mapping, 'bst', default_value={})
         element_splits = _yaml.node_get(element_bst, Mapping, 'split-rules', default_value={})
 
-        # Extend project wide split rules with any split rules defined by the element
-        _yaml.composite(project_splits, element_splits)
+        if self.__is_junction:
+            splits = _yaml.node_chain_copy(element_splits)
+        else:
+            assert project._splits is not None
+
+            splits = _yaml.node_chain_copy(project._splits)
+            # Extend project wide split rules with any split rules defined by the element
+            _yaml.composite(splits, element_splits)
 
-        element_bst['split-rules'] = project_splits
+        element_bst['split-rules'] = splits
         element_public['bst'] = element_bst
         defaults['public'] = element_public
 
@@ -2199,7 +2213,11 @@ class Element(Plugin):
             # Override the element's defaults with element specific
             # overrides from the project.conf
             project = self._get_project()
-            elements = project.element_overrides
+            if self.__is_junction:
+                elements = project.first_pass_config.element_overrides
+            else:
+                elements = project.element_overrides
+
             overrides = elements.get(self.get_kind())
             if overrides:
                 _yaml.composite(defaults, overrides)
@@ -2212,10 +2230,14 @@ class Element(Plugin):
     # creating sandboxes for this element
     #
     def __extract_environment(self, meta):
-        project = self._get_project()
         default_env = _yaml.node_get(self.__defaults, Mapping, 'environment', default_value={})
 
-        environment = _yaml.node_chain_copy(project.base_environment)
+        if self.__is_junction:
+            environment = {}
+        else:
+            project = self._get_project()
+            environment = _yaml.node_chain_copy(project.base_environment)
+
         _yaml.composite(environment, default_env)
         _yaml.composite(environment, meta.environment)
         _yaml.node_final_assertions(environment)
@@ -2228,8 +2250,13 @@ class Element(Plugin):
         return final_env
 
     def __extract_env_nocache(self, meta):
-        project = self._get_project()
-        project_nocache = project.base_env_nocache
+        if self.__is_junction:
+            project_nocache = []
+        else:
+            project = self._get_project()
+            project.ensure_fully_loaded()
+            project_nocache = project.base_env_nocache
+
         default_nocache = _yaml.node_get(self.__defaults, list, 'environment-nocache', default_value=[])
         element_nocache = meta.env_nocache
 
@@ -2244,10 +2271,15 @@ class Element(Plugin):
     # substituting command strings to be run in the sandbox
     #
     def __extract_variables(self, meta):
-        project = self._get_project()
         default_vars = _yaml.node_get(self.__defaults, Mapping, 'variables', default_value={})
 
-        variables = _yaml.node_chain_copy(project.base_variables)
+        project = self._get_project()
+        if self.__is_junction:
+            variables = _yaml.node_chain_copy(project.first_pass_config.base_variables)
+        else:
+            project.ensure_fully_loaded()
+            variables = _yaml.node_chain_copy(project.base_variables)
+
         _yaml.composite(variables, default_vars)
         _yaml.composite(variables, meta.variables)
         _yaml.node_final_assertions(variables)
@@ -2271,13 +2303,18 @@ class Element(Plugin):
     # Sandbox-specific configuration data, to be passed to the sandbox's constructor.
     #
     def __extract_sandbox_config(self, meta):
-        project = self._get_project()
+        if self.__is_junction:
+            sandbox_config = {'build-uid': 0,
+                              'build-gid': 0}
+        else:
+            project = self._get_project()
+            project.ensure_fully_loaded()
+            sandbox_config = _yaml.node_chain_copy(project._sandbox)
 
         # The default config is already composited with the project overrides
         sandbox_defaults = _yaml.node_get(self.__defaults, Mapping, 'sandbox', default_value={})
         sandbox_defaults = _yaml.node_chain_copy(sandbox_defaults)
 
-        sandbox_config = _yaml.node_chain_copy(project._sandbox)
         _yaml.composite(sandbox_config, sandbox_defaults)
         _yaml.composite(sandbox_config, meta.sandbox)
         _yaml.node_final_assertions(sandbox_config)
diff --git a/buildstream/source.py b/buildstream/source.py
index 2f3f1c281..d58bfe2a3 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -227,8 +227,10 @@ class Source(Plugin):
 
         # Collect the composited element configuration and
         # ask the element to configure itself.
-        self.__init_defaults()
+        self.__init_defaults(meta)
         self.__config = self.__extract_config(meta)
+        self.__first_pass = meta.first_pass
+
         self.configure(self.__config)
 
     COMMON_CONFIG_KEYS = ['kind', 'directory']
@@ -454,7 +456,7 @@ class Source(Plugin):
                 self.__expected_alias = url_alias
 
             project = self._get_project()
-            return project.translate_url(url)
+            return project.translate_url(url, first_pass=self.__first_pass)
 
     def get_project_directory(self):
         """Fetch the project base directory
@@ -524,7 +526,7 @@ class Source(Plugin):
             for fetcher in source_fetchers:
                 alias = fetcher._get_alias()
                 success = False
-                for uri in project.get_alias_uris(alias):
+                for uri in project.get_alias_uris(alias, first_pass=self.__first_pass):
                     try:
                         fetcher.fetch(uri)
                     # FIXME: Need to consider temporary vs. permanent failures,
@@ -538,13 +540,17 @@ class Source(Plugin):
                     raise last_error
         else:
             alias = self._get_alias()
-            if not project.mirrors or not alias:
+            if self.__first_pass:
+                mirrors = project.first_pass_config.mirrors
+            else:
+                mirrors = project.config.mirrors
+            if not mirrors or not alias:
                 self.fetch()
                 return
 
             context = self._get_context()
             source_kind = type(self)
-            for uri in project.get_alias_uris(alias):
+            for uri in project.get_alias_uris(alias, first_pass=self.__first_pass):
                 new_source = source_kind(context, project, self.__meta,
                                          alias_override=(alias, uri))
                 new_source._preflight()
@@ -739,24 +745,29 @@ class Source(Plugin):
         #
         # Step 3 - Apply the change in project data
         #
-        if project is toplevel:
-            if toplevel.ref_storage == ProjectRefStorage.PROJECT_REFS:
-                do_save_refs(toplevel_refs)
-            else:
+        if toplevel.ref_storage == ProjectRefStorage.PROJECT_REFS:
+            do_save_refs(toplevel_refs)
+        else:
+            if provenance.filename.project is toplevel:
                 # Save the ref in the originating file
                 #
-                fullname = os.path.join(toplevel.element_path, provenance.filename)
                 try:
-                    _yaml.dump(provenance.toplevel, fullname)
+                    _yaml.dump(_yaml.node_sanitize(provenance.toplevel), provenance.filename.name)
                 except OSError as e:
                     raise SourceError("{}: Error saving source reference to '{}': {}"
-                                      .format(self, provenance.filename, e),
+                                      .format(self, provenance.filename.name, e),
                                       reason="save-ref-error") from e
-        else:
-            if toplevel.ref_storage == ProjectRefStorage.PROJECT_REFS:
-                do_save_refs(toplevel_refs)
-            else:
+            elif provenance.filename.project is project:
                 self.warn("{}: Not persisting new reference in junctioned project".format(self))
+            elif provenance.filename.project is None:
+                assert provenance.filename.name == ''
+                assert provenance.filename.shortname == ''
+                raise SourceError("{}: Error saving source reference to synthetic node."
+                                  .format(self))
+            else:
+                raise SourceError("{}: Cannot track source in a fragment from a junction"
+                                  .format(provenance.filename.shortname),
+                                  reason="tracking-junction-fragment")
 
         return changed
 
@@ -779,7 +790,7 @@ class Source(Plugin):
     def _get_alias(self):
         alias = self.__expected_alias
         project = self._get_project()
-        if project.get_alias_uri(alias):
+        if project.get_alias_uri(alias, first_pass=self.__first_pass):
             # The alias must already be defined in the project's aliases
             # otherwise http://foo gets treated like it contains an alias
             return alias
@@ -795,7 +806,11 @@ class Source(Plugin):
         project = self._get_project()
         # If there are no mirrors, or no aliases to replace, there's nothing to do here.
         alias = self._get_alias()
-        if not project.mirrors or not alias:
+        if self.__first_pass:
+            mirrors = project.first_pass_config.mirrors
+        else:
+            mirrors = project.config.mirrors
+        if not mirrors or not alias:
             return self.track()
 
         context = self._get_context()
@@ -803,7 +818,7 @@ class Source(Plugin):
 
         # NOTE: We are assuming here that tracking only requires substituting the
         #       first alias used
-        for uri in reversed(project.get_alias_uris(alias)):
+        for uri in reversed(project.get_alias_uris(alias, first_pass=self.__first_pass)):
             new_source = source_kind(context, project, self.__meta,
                                      alias_override=(alias, uri))
             new_source._preflight()
@@ -831,10 +846,13 @@ class Source(Plugin):
                               reason="ensure-stage-dir-fail") from e
         return directory
 
-    def __init_defaults(self):
+    def __init_defaults(self, meta):
         if not self.__defaults_set:
             project = self._get_project()
-            sources = project.source_overrides
+            if meta.first_pass:
+                sources = project.first_pass_config.source_overrides
+            else:
+                sources = project.source_overrides
             type(self).__defaults = sources.get(self.get_kind(), {})
             type(self).__defaults_set = True