Move source from 'buildstream' to 'src/buildstream'

This was discussed in #1008.

Fixes #1009.

6 files changed, 1127 insertions, 0 deletions

diff --git a/src/buildstream/_loader/__init__.py b/src/buildstream/_loader/__init__.py
new file mode 100644
index 000000000..a2c31796e
--- /dev/null
+++ b/src/buildstream/_loader/__init__.py
@@ -0,0 +1,22 @@
+#
+#  Copyright (C) 2018 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+from .metasource import MetaSource
+from .metaelement import MetaElement
+from .loader import Loader
diff --git a/src/buildstream/_loader/loadelement.py b/src/buildstream/_loader/loadelement.py
new file mode 100644
index 000000000..684c32554
--- /dev/null
+++ b/src/buildstream/_loader/loadelement.py
@@ -0,0 +1,181 @@
+#
+#  Copyright (C) 2016 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+# System imports
+from itertools import count
+
+from pyroaring import BitMap, FrozenBitMap  # pylint: disable=no-name-in-module
+
+# BuildStream toplevel imports
+from .. import _yaml
+
+# Local package imports
+from .types import Symbol, Dependency
+
+
+# LoadElement():
+#
+# A transient object breaking down what is loaded allowing us to
+# do complex operations in multiple passes.
+#
+# Args:
+#    node (dict): A YAML loaded dictionary
+#    name (str): The element name
+#    loader (Loader): The Loader object for this element
+#
+class LoadElement():
+    # Dependency():
+    #
+    # A link from a LoadElement to its dependencies.
+    #
+    # Keeps a link to one of the current Element's dependencies, together with
+    # its dependency type.
+    #
+    # Args:
+    #    element (LoadElement): a LoadElement on which there is a dependency
+    #    dep_type (str): the type of dependency this dependency link is
+    class Dependency:
+        def __init__(self, element, dep_type):
+            self.element = element
+            self.dep_type = dep_type
+
+    _counter = count()
+
+    def __init__(self, node, filename, loader):
+
+        #
+        # Public members
+        #
+        self.node = node       # The YAML node
+        self.name = filename   # The element name
+        self.full_name = None  # The element full name (with associated junction)
+        self.deps = None       # The list of Dependency objects
+        self.node_id = next(self._counter)
+
+        #
+        # Private members
+        #
+        self._loader = loader   # The Loader object
+        self._dep_cache = None  # The dependency cache, to speed up depends()
+
+        #
+        # Initialization
+        #
+        if loader.project.junction:
+            # dependency is in subproject, qualify name
+            self.full_name = '{}:{}'.format(loader.project.junction.name, self.name)
+        else:
+            # dependency is in top-level project
+            self.full_name = self.name
+
+        # Ensure the root node is valid
+        _yaml.node_validate(self.node, [
+            'kind', 'depends', 'sources', 'sandbox',
+            'variables', 'environment', 'environment-nocache',
+            'config', 'public', 'description',
+            'build-depends', 'runtime-depends',
+        ])
+
+        self.dependencies = []
+
+    @property
+    def junction(self):
+        return self._loader.project.junction
+
+    # depends():
+    #
+    # Checks if this element depends on another element, directly
+    # or indirectly.
+    #
+    # Args:
+    #    other (LoadElement): Another LoadElement
+    #
+    # Returns:
+    #    (bool): True if this LoadElement depends on 'other'
+    #
+    def depends(self, other):
+        self._ensure_depends_cache()
+        return other.node_id in self._dep_cache
+
+    ###########################################
+    #            Private Methods              #
+    ###########################################
+    def _ensure_depends_cache(self):
+
+        if self._dep_cache:
+            return
+
+        self._dep_cache = BitMap()
+
+        for dep in self.dependencies:
+            elt = dep.element
+
+            # Ensure the cache of the element we depend on
+            elt._ensure_depends_cache()
+
+            # We depend on this element
+            self._dep_cache.add(elt.node_id)
+
+            # And we depend on everything this element depends on
+            self._dep_cache.update(elt._dep_cache)
+
+        self._dep_cache = FrozenBitMap(self._dep_cache)
+
+
+# _extract_depends_from_node():
+#
+# Creates an array of Dependency objects from a given dict node 'node',
+# allows both strings and dicts for expressing the dependency and
+# throws a comprehensive LoadError in the case that the node is malformed.
+#
+# After extracting depends, the symbol is deleted from the node
+#
+# Args:
+#    node (dict): A YAML loaded dictionary
+#
+# Returns:
+#    (list): a list of Dependency objects
+#
+def _extract_depends_from_node(node, *, key=None):
+    if key is None:
+        build_depends = _extract_depends_from_node(node, key=Symbol.BUILD_DEPENDS)
+        runtime_depends = _extract_depends_from_node(node, key=Symbol.RUNTIME_DEPENDS)
+        depends = _extract_depends_from_node(node, key=Symbol.DEPENDS)
+        return build_depends + runtime_depends + depends
+    elif key == Symbol.BUILD_DEPENDS:
+        default_dep_type = Symbol.BUILD
+    elif key == Symbol.RUNTIME_DEPENDS:
+        default_dep_type = Symbol.RUNTIME
+    elif key == Symbol.DEPENDS:
+        default_dep_type = None
+    else:
+        assert False, "Unexpected value of key '{}'".format(key)
+
+    depends = _yaml.node_get(node, list, key, default_value=[])
+    output_deps = []
+
+    for index, dep in enumerate(depends):
+        dep_provenance = _yaml.node_get_provenance(node, key=key, indices=[index])
+        dependency = Dependency(dep, dep_provenance, default_dep_type=default_dep_type)
+        output_deps.append(dependency)
+
+    # Now delete the field, we dont want it anymore
+    _yaml.node_del(node, key, safe=True)
+
+    return output_deps
diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py
new file mode 100644
index 000000000..261ec40e4
--- /dev/null
+++ b/src/buildstream/_loader/loader.py
@@ -0,0 +1,710 @@
+#
+#  Copyright (C) 2018 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+import os
+from functools import cmp_to_key
+from collections.abc import Mapping
+
+from .._exceptions import LoadError, LoadErrorReason
+from .. import Consistency
+from .. import _yaml
+from ..element import Element
+from .._profile import Topics, PROFILER
+from .._includes import Includes
+
+from .types import Symbol
+from .loadelement import LoadElement, _extract_depends_from_node
+from .metaelement import MetaElement
+from .metasource import MetaSource
+from ..types import CoreWarnings
+from .._message import Message, MessageType
+
+
+# Loader():
+#
+# The Loader class does the heavy lifting of parsing target
+# bst files and ultimately transforming them into a list of MetaElements
+# with their own MetaSources, ready for instantiation by the core.
+#
+# Args:
+#    context (Context): The Context object
+#    project (Project): The toplevel Project object
+#    parent (Loader): A parent Loader object, in the case this is a junctioned Loader
+#
+class Loader():
+
+    def __init__(self, context, project, *, parent=None):
+
+        # Ensure we have an absolute path for the base directory
+        basedir = project.element_path
+        if not os.path.isabs(basedir):
+            basedir = os.path.abspath(basedir)
+
+        #
+        # Public members
+        #
+        self.project = project   # The associated Project
+
+        #
+        # Private members
+        #
+        self._context = context
+        self._options = project.options      # Project options (OptionPool)
+        self._basedir = basedir              # Base project directory
+        self._first_pass_options = project.first_pass_config.options  # Project options (OptionPool)
+        self._parent = parent                # The parent loader
+
+        self._meta_elements = {}  # Dict of resolved meta elements by name
+        self._elements = {}       # Dict of elements
+        self._loaders = {}        # Dict of junction loaders
+
+        self._includes = Includes(self, copy_tree=True)
+
+    # load():
+    #
+    # Loads the project based on the parameters given to the constructor
+    #
+    # Args:
+    #    rewritable (bool): Whether the loaded files should be rewritable
+    #                       this is a bit more expensive due to deep copies
+    #    ticker (callable): An optional function for tracking load progress
+    #    targets (list of str): Target, element-path relative bst filenames in the project
+    #    fetch_subprojects (bool): Whether to fetch subprojects while loading
+    #
+    # Raises: LoadError
+    #
+    # Returns: The toplevel LoadElement
+    def load(self, targets, rewritable=False, ticker=None, fetch_subprojects=False):
+
+        for filename in targets:
+            if os.path.isabs(filename):
+                # XXX Should this just be an assertion ?
+                # Expect that the caller gives us the right thing at least ?
+                raise LoadError(LoadErrorReason.INVALID_DATA,
+                                "Target '{}' was not specified as a relative "
+                                "path to the base project directory: {}"
+                                .format(filename, self._basedir))
+
+        self._warn_invalid_elements(targets)
+
+        # First pass, recursively load files and populate our table of LoadElements
+        #
+        target_elements = []
+
+        for target in targets:
+            with PROFILER.profile(Topics.LOAD_PROJECT, target):
+                _junction, name, loader = self._parse_name(target, rewritable, ticker,
+                                                           fetch_subprojects=fetch_subprojects)
+                element = loader._load_file(name, rewritable, ticker, fetch_subprojects)
+                target_elements.append(element)
+
+        #
+        # Now that we've resolve the dependencies, scan them for circular dependencies
+        #
+
+        # Set up a dummy element that depends on all top-level targets
+        # to resolve potential circular dependencies between them
+        dummy_target = LoadElement(_yaml.new_empty_node(), "", self)
+        dummy_target.dependencies.extend(
+            LoadElement.Dependency(element, Symbol.RUNTIME)
+            for element in target_elements
+        )
+
+        with PROFILER.profile(Topics.CIRCULAR_CHECK, "_".join(targets)):
+            self._check_circular_deps(dummy_target)
+
+        ret = []
+        #
+        # Sort direct dependencies of elements by their dependency ordering
+        #
+        for element in target_elements:
+            loader = element._loader
+            with PROFILER.profile(Topics.SORT_DEPENDENCIES, element.name):
+                loader._sort_dependencies(element)
+
+            # Finally, wrap what we have into LoadElements and return the target
+            #
+            ret.append(loader._collect_element(element))
+
+        self._clean_caches()
+
+        return ret
+
+    # clean_caches()
+    #
+    # Clean internal loader caches, recursively
+    #
+    # When loading the elements, the loaders use caches in order to not load the
+    # same element twice. These are kept after loading and prevent garbage
+    # collection. Cleaning them explicitely is required.
+    #
+    def _clean_caches(self):
+        for loader in self._loaders.values():
+            # value may be None with nested junctions without overrides
+            if loader is not None:
+                loader._clean_caches()
+
+        self._meta_elements = {}
+        self._elements = {}
+
+    ###########################################
+    #            Private Methods              #
+    ###########################################
+
+    # _load_file():
+    #
+    # Recursively load bst files
+    #
+    # Args:
+    #    filename (str): The element-path relative bst file
+    #    rewritable (bool): Whether we should load in round trippable mode
+    #    ticker (callable): A callback to report loaded filenames to the frontend
+    #    fetch_subprojects (bool): Whether to fetch subprojects while loading
+    #    provenance (Provenance): The location from where the file was referred to, or None
+    #
+    # Returns:
+    #    (LoadElement): A loaded LoadElement
+    #
+    def _load_file(self, filename, rewritable, ticker, fetch_subprojects, provenance=None):
+
+        # Silently ignore already loaded files
+        if filename in self._elements:
+            return self._elements[filename]
+
+        # Call the ticker
+        if ticker:
+            ticker(filename)
+
+        # Load the data and process any conditional statements therein
+        fullpath = os.path.join(self._basedir, filename)
+        try:
+            node = _yaml.load(fullpath, shortname=filename, copy_tree=rewritable,
+                              project=self.project)
+        except LoadError as e:
+            if e.reason == LoadErrorReason.MISSING_FILE:
+
+                if self.project.junction:
+                    message = "Could not find element '{}' in project referred to by junction element '{}'" \
+                              .format(filename, self.project.junction.name)
+                else:
+                    message = "Could not find element '{}' in elements directory '{}'".format(filename, self._basedir)
+
+                if provenance:
+                    message = "{}: {}".format(provenance, message)
+
+                # If we can't find the file, try to suggest plausible
+                # alternatives by stripping the element-path from the given
+                # filename, and verifying that it exists.
+                detail = None
+                elements_dir = os.path.relpath(self._basedir, self.project.directory)
+                element_relpath = os.path.relpath(filename, elements_dir)
+                if filename.startswith(elements_dir) and os.path.exists(os.path.join(self._basedir, element_relpath)):
+                    detail = "Did you mean '{}'?".format(element_relpath)
+
+                raise LoadError(LoadErrorReason.MISSING_FILE,
+                                message, detail=detail) from e
+
+            elif e.reason == LoadErrorReason.LOADING_DIRECTORY:
+                # If a <directory>.bst file exists in the element path,
+                # let's suggest this as a plausible alternative.
+                message = str(e)
+                if provenance:
+                    message = "{}: {}".format(provenance, message)
+                detail = None
+                if os.path.exists(os.path.join(self._basedir, filename + '.bst')):
+                    element_name = filename + '.bst'
+                    detail = "Did you mean '{}'?\n".format(element_name)
+                raise LoadError(LoadErrorReason.LOADING_DIRECTORY,
+                                message, detail=detail) from e
+            else:
+                raise
+        kind = _yaml.node_get(node, str, Symbol.KIND)
+        if kind == "junction":
+            self._first_pass_options.process_node(node)
+        else:
+            self.project.ensure_fully_loaded()
+
+            self._includes.process(node)
+
+            self._options.process_node(node)
+
+        element = LoadElement(node, filename, self)
+
+        self._elements[filename] = element
+
+        dependencies = _extract_depends_from_node(node)
+
+        # Load all dependency files for the new LoadElement
+        for dep in dependencies:
+            if dep.junction:
+                self._load_file(dep.junction, rewritable, ticker, fetch_subprojects, dep.provenance)
+                loader = self._get_loader(dep.junction, rewritable=rewritable, ticker=ticker,
+                                          fetch_subprojects=fetch_subprojects, provenance=dep.provenance)
+            else:
+                loader = self
+
+            dep_element = loader._load_file(dep.name, rewritable, ticker,
+                                            fetch_subprojects, dep.provenance)
+
+            if _yaml.node_get(dep_element.node, str, Symbol.KIND) == 'junction':
+                raise LoadError(LoadErrorReason.INVALID_DATA,
+                                "{}: Cannot depend on junction"
+                                .format(dep.provenance))
+
+            element.dependencies.append(LoadElement.Dependency(dep_element, dep.dep_type))
+
+        deps_names = [dep.name for dep in dependencies]
+        self._warn_invalid_elements(deps_names)
+
+        return element
+
+    # _check_circular_deps():
+    #
+    # Detect circular dependencies on LoadElements with
+    # dependencies already resolved.
+    #
+    # Args:
+    #    element (str): The element to check
+    #
+    # Raises:
+    #    (LoadError): In case there was a circular dependency error
+    #
+    def _check_circular_deps(self, element, check_elements=None, validated=None, sequence=None):
+
+        if check_elements is None:
+            check_elements = set()
+        if validated is None:
+            validated = set()
+        if sequence is None:
+            sequence = []
+
+        # Skip already validated branches
+        if element in validated:
+            return
+
+        if element in check_elements:
+            # Create `chain`, the loop of element dependencies from this
+            # element back to itself, by trimming everything before this
+            # element from the sequence under consideration.
+            chain = sequence[sequence.index(element.full_name):]
+            chain.append(element.full_name)
+            raise LoadError(LoadErrorReason.CIRCULAR_DEPENDENCY,
+                            ("Circular dependency detected at element: {}\n" +
+                             "Dependency chain: {}")
+                            .format(element.full_name, " -> ".join(chain)))
+
+        # Push / Check each dependency / Pop
+        check_elements.add(element)
+        sequence.append(element.full_name)
+        for dep in element.dependencies:
+            dep.element._loader._check_circular_deps(dep.element, check_elements, validated, sequence)
+        check_elements.remove(element)
+        sequence.pop()
+
+        # Eliminate duplicate paths
+        validated.add(element)
+
+    # _sort_dependencies():
+    #
+    # Sort dependencies of each element by their dependencies,
+    # so that direct dependencies which depend on other direct
+    # dependencies (directly or indirectly) appear later in the
+    # list.
+    #
+    # This avoids the need for performing multiple topological
+    # sorts throughout the build process.
+    #
+    # Args:
+    #    element (LoadElement): The element to sort
+    #
+    def _sort_dependencies(self, element, visited=None):
+        if visited is None:
+            visited = set()
+
+        if element in visited:
+            return
+
+        for dep in element.dependencies:
+            dep.element._loader._sort_dependencies(dep.element, visited=visited)
+
+        def dependency_cmp(dep_a, dep_b):
+            element_a = dep_a.element
+            element_b = dep_b.element
+
+            # Sort on inter element dependency first
+            if element_a.depends(element_b):
+                return 1
+            elif element_b.depends(element_a):
+                return -1
+
+            # If there are no inter element dependencies, place
+            # runtime only dependencies last
+            if dep_a.dep_type != dep_b.dep_type:
+                if dep_a.dep_type == Symbol.RUNTIME:
+                    return 1
+                elif dep_b.dep_type == Symbol.RUNTIME:
+                    return -1
+
+            # All things being equal, string comparison.
+            if element_a.name > element_b.name:
+                return 1
+            elif element_a.name < element_b.name:
+                return -1
+
+            # Sort local elements before junction elements
+            # and use string comparison between junction elements
+            if element_a.junction and element_b.junction:
+                if element_a.junction > element_b.junction:
+                    return 1
+                elif element_a.junction < element_b.junction:
+                    return -1
+            elif element_a.junction:
+                return -1
+            elif element_b.junction:
+                return 1
+
+            # This wont ever happen
+            return 0
+
+        # Now dependency sort, we ensure that if any direct dependency
+        # directly or indirectly depends on another direct dependency,
+        # it is found later in the list.
+        element.dependencies.sort(key=cmp_to_key(dependency_cmp))
+
+        visited.add(element)
+
+    # _collect_element()
+    #
+    # Collect the toplevel elements we have
+    #
+    # Args:
+    #    element (LoadElement): The element for which to load a MetaElement
+    #
+    # Returns:
+    #    (MetaElement): A recursively loaded MetaElement
+    #
+    def _collect_element(self, element):
+        # Return the already built one, if we already built it
+        meta_element = self._meta_elements.get(element.name)
+        if meta_element:
+            return meta_element
+
+        node = element.node
+        elt_provenance = _yaml.node_get_provenance(node)
+        meta_sources = []
+
+        sources = _yaml.node_get(node, list, Symbol.SOURCES, default_value=[])
+        element_kind = _yaml.node_get(node, str, Symbol.KIND)
+
+        # Safe loop calling into _yaml.node_get() for each element ensures
+        # we have good error reporting
+        for i in range(len(sources)):
+            source = _yaml.node_get(node, Mapping, Symbol.SOURCES, indices=[i])
+            kind = _yaml.node_get(source, str, Symbol.KIND)
+            _yaml.node_del(source, Symbol.KIND)
+
+            # Directory is optional
+            directory = _yaml.node_get(source, str, Symbol.DIRECTORY, default_value=None)
+            if directory:
+                _yaml.node_del(source, Symbol.DIRECTORY)
+
+            index = sources.index(source)
+            meta_source = MetaSource(element.name, index, element_kind, kind, source, directory)
+            meta_sources.append(meta_source)
+
+        meta_element = MetaElement(self.project, element.name, element_kind,
+                                   elt_provenance, meta_sources,
+                                   _yaml.node_get(node, Mapping, Symbol.CONFIG, default_value={}),
+                                   _yaml.node_get(node, Mapping, Symbol.VARIABLES, default_value={}),
+                                   _yaml.node_get(node, Mapping, Symbol.ENVIRONMENT, default_value={}),
+                                   _yaml.node_get(node, list, Symbol.ENV_NOCACHE, default_value=[]),
+                                   _yaml.node_get(node, Mapping, Symbol.PUBLIC, default_value={}),
+                                   _yaml.node_get(node, Mapping, Symbol.SANDBOX, default_value={}),
+                                   element_kind == 'junction')
+
+        # Cache it now, make sure it's already there before recursing
+        self._meta_elements[element.name] = meta_element
+
+        # Descend
+        for dep in element.dependencies:
+            loader = dep.element._loader
+            meta_dep = loader._collect_element(dep.element)
+            if dep.dep_type != 'runtime':
+                meta_element.build_dependencies.append(meta_dep)
+            if dep.dep_type != 'build':
+                meta_element.dependencies.append(meta_dep)
+
+        return meta_element
+
+    # _get_loader():
+    #
+    # Return loader for specified junction
+    #
+    # Args:
+    #    filename (str): Junction name
+    #    fetch_subprojects (bool): Whether to fetch subprojects while loading
+    #
+    # Raises: LoadError
+    #
+    # Returns: A Loader or None if specified junction does not exist
+    def _get_loader(self, filename, *, rewritable=False, ticker=None, level=0,
+                    fetch_subprojects=False, provenance=None):
+
+        provenance_str = ""
+        if provenance is not None:
+            provenance_str = "{}: ".format(provenance)
+
+        # return previously determined result
+        if filename in self._loaders:
+            loader = self._loaders[filename]
+
+            if loader is None:
+                # do not allow junctions with the same name in different
+                # subprojects
+                raise LoadError(LoadErrorReason.CONFLICTING_JUNCTION,
+                                "{}Conflicting junction {} in subprojects, define junction in {}"
+                                .format(provenance_str, filename, self.project.name))
+
+            return loader
+
+        if self._parent:
+            # junctions in the parent take precedence over junctions defined
+            # in subprojects
+            loader = self._parent._get_loader(filename, rewritable=rewritable, ticker=ticker,
+                                              level=level + 1, fetch_subprojects=fetch_subprojects,
+                                              provenance=provenance)
+            if loader:
+                self._loaders[filename] = loader
+                return loader
+
+        try:
+            self._load_file(filename, rewritable, ticker, fetch_subprojects)
+        except LoadError as e:
+            if e.reason != LoadErrorReason.MISSING_FILE:
+                # other load error
+                raise
+
+            if level == 0:
+                # junction element not found in this or ancestor projects
+                raise
+            else:
+                # mark junction as not available to allow detection of
+                # conflicting junctions in subprojects
+                self._loaders[filename] = None
+                return None
+
+        # meta junction element
+        meta_element = self._collect_element(self._elements[filename])
+        if meta_element.kind != 'junction':
+            raise LoadError(LoadErrorReason.INVALID_DATA,
+                            "{}{}: Expected junction but element kind is {}".format(
+                                provenance_str, filename, meta_element.kind))
+
+        element = Element._new_from_meta(meta_element)
+        element._preflight()
+
+        # If this junction element points to a sub-sub-project, we need to
+        # find loader for that project.
+        if element.target:
+            subproject_loader = self._get_loader(element.target_junction, rewritable=rewritable, ticker=ticker,
+                                                 level=level, fetch_subprojects=fetch_subprojects,
+                                                 provenance=provenance)
+            loader = subproject_loader._get_loader(element.target_element, rewritable=rewritable, ticker=ticker,
+                                                   level=level, fetch_subprojects=fetch_subprojects,
+                                                   provenance=provenance)
+            self._loaders[filename] = loader
+            return loader
+
+        sources = list(element.sources())
+        if not element._source_cached():
+            for idx, source in enumerate(sources):
+                # Handle the case where a subproject needs to be fetched
+                #
+                if source.get_consistency() == Consistency.RESOLVED:
+                    if fetch_subprojects:
+                        if ticker:
+                            ticker(filename, 'Fetching subproject from {} source'.format(source.get_kind()))
+                        source._fetch(sources[0:idx])
+                    else:
+                        detail = "Try fetching the project with `bst source fetch {}`".format(filename)
+                        raise LoadError(LoadErrorReason.SUBPROJECT_FETCH_NEEDED,
+                                        "{}Subproject fetch needed for junction: {}".format(provenance_str, filename),
+                                        detail=detail)
+
+                # Handle the case where a subproject has no ref
+                #
+                elif source.get_consistency() == Consistency.INCONSISTENT:
+                    detail = "Try tracking the junction element with `bst source track {}`".format(filename)
+                    raise LoadError(LoadErrorReason.SUBPROJECT_INCONSISTENT,
+                                    "{}Subproject has no ref for junction: {}".format(provenance_str, filename),
+                                    detail=detail)
+
+        workspace = element._get_workspace()
+        if workspace:
+            # If a workspace is open, load it from there instead
+            basedir = workspace.get_absolute_path()
+        elif len(sources) == 1 and sources[0]._get_local_path():
+            # Optimization for junctions with a single local source
+            basedir = sources[0]._get_local_path()
+        else:
+            # Stage sources
+            element._set_required()
+            basedir = os.path.join(self.project.directory, ".bst", "staged-junctions",
+                                   filename, element._get_cache_key())
+            if not os.path.exists(basedir):
+                os.makedirs(basedir, exist_ok=True)
+                element._stage_sources_at(basedir, mount_workspaces=False)
+
+        # Load the project
+        project_dir = os.path.join(basedir, element.path)
+        try:
+            from .._project import Project  # pylint: disable=cyclic-import
+            project = Project(project_dir, self._context, junction=element,
+                              parent_loader=self, search_for_project=False)
+        except LoadError as e:
+            if e.reason == LoadErrorReason.MISSING_PROJECT_CONF:
+                message = (
+                    provenance_str + "Could not find the project.conf file in the project "
+                    "referred to by junction element '{}'.".format(element.name)
+                )
+                if element.path:
+                    message += " Was expecting it at path '{}' in the junction's source.".format(element.path)
+                raise LoadError(reason=LoadErrorReason.INVALID_JUNCTION,
+                                message=message) from e
+            else:
+                raise
+
+        loader = project.loader
+        self._loaders[filename] = loader
+
+        return loader
+
+    # _parse_name():
+    #
+    # Get junction and base name of element along with loader for the sub-project
+    #
+    # Args:
+    #   name (str): Name of target
+    #   rewritable (bool): Whether the loaded files should be rewritable
+    #                      this is a bit more expensive due to deep copies
+    #   ticker (callable): An optional function for tracking load progress
+    #   fetch_subprojects (bool): Whether to fetch subprojects while loading
+    #
+    # Returns:
+    #   (tuple): - (str): name of the junction element
+    #            - (str): name of the element
+    #            - (Loader): loader for sub-project
+    #
+    def _parse_name(self, name, rewritable, ticker, fetch_subprojects=False):
+        # We allow to split only once since deep junctions names are forbidden.
+        # Users who want to refer to elements in sub-sub-projects are required
+        # to create junctions on the top level project.
+        junction_path = name.rsplit(':', 1)
+        if len(junction_path) == 1:
+            return None, junction_path[-1], self
+        else:
+            self._load_file(junction_path[-2], rewritable, ticker, fetch_subprojects)
+            loader = self._get_loader(junction_path[-2], rewritable=rewritable, ticker=ticker,
+                                      fetch_subprojects=fetch_subprojects)
+            return junction_path[-2], junction_path[-1], loader
+
+    # Print a warning message, checks warning_token against project configuration
+    #
+    # Args:
+    #     brief (str): The brief message
+    #     warning_token (str): An optional configurable warning assosciated with this warning,
+    #                          this will cause PluginError to be raised if this warning is configured as fatal.
+    #                          (*Since 1.4*)
+    #
+    # Raises:
+    #     (:class:`.LoadError`): When warning_token is considered fatal by the project configuration
+    #
+    def _warn(self, brief, *, warning_token=None):
+        if warning_token:
+            if self.project._warning_is_fatal(warning_token):
+                raise LoadError(warning_token, brief)
+
+        message = Message(None, MessageType.WARN, brief)
+        self._context.message(message)
+
+    # Print warning messages if any of the specified elements have invalid names.
+    #
+    # Valid filenames should end with ".bst" extension.
+    #
+    # Args:
+    #    elements (list): List of element names
+    #
+    # Raises:
+    #     (:class:`.LoadError`): When warning_token is considered fatal by the project configuration
+    #
+    def _warn_invalid_elements(self, elements):
+
+        # invalid_elements
+        #
+        # A dict that maps warning types to the matching elements.
+        invalid_elements = {
+            CoreWarnings.BAD_ELEMENT_SUFFIX: [],
+            CoreWarnings.BAD_CHARACTERS_IN_NAME: [],
+        }
+
+        for filename in elements:
+            if not filename.endswith(".bst"):
+                invalid_elements[CoreWarnings.BAD_ELEMENT_SUFFIX].append(filename)
+            if not self._valid_chars_name(filename):
+                invalid_elements[CoreWarnings.BAD_CHARACTERS_IN_NAME].append(filename)
+
+        if invalid_elements[CoreWarnings.BAD_ELEMENT_SUFFIX]:
+            self._warn("Target elements '{}' do not have expected file extension `.bst` "
+                       "Improperly named elements will not be discoverable by commands"
+                       .format(invalid_elements[CoreWarnings.BAD_ELEMENT_SUFFIX]),
+                       warning_token=CoreWarnings.BAD_ELEMENT_SUFFIX)
+        if invalid_elements[CoreWarnings.BAD_CHARACTERS_IN_NAME]:
+            self._warn("Target elements '{}' have invalid characerts in their name."
+                       .format(invalid_elements[CoreWarnings.BAD_CHARACTERS_IN_NAME]),
+                       warning_token=CoreWarnings.BAD_CHARACTERS_IN_NAME)
+
+    # Check if given filename containers valid characters.
+    #
+    # Args:
+    #    name (str): Name of the file
+    #
+    # Returns:
+    #    (bool): True if all characters are valid, False otherwise.
+    #
+    def _valid_chars_name(self, name):
+        for char in name:
+            char_val = ord(char)
+
+            # 0-31 are control chars, 127 is DEL, and >127 means non-ASCII
+            if char_val <= 31 or char_val >= 127:
+                return False
+
+            # Disallow characters that are invalid on Windows. The list can be
+            # found at https://docs.microsoft.com/en-us/windows/desktop/FileIO/naming-a-file
+            #
+            # Note that although : (colon) is not allowed, we do not raise
+            # warnings because of that, since we use it as a separator for
+            # junctioned elements.
+            #
+            # We also do not raise warnings on slashes since they are used as
+            # path separators.
+            if char in r'<>"|?*':
+                return False
+
+        return True
diff --git a/src/buildstream/_loader/metaelement.py b/src/buildstream/_loader/metaelement.py
new file mode 100644
index 000000000..45eb6f4d0
--- /dev/null
+++ b/src/buildstream/_loader/metaelement.py
@@ -0,0 +1,60 @@
+#
+#  Copyright (C) 2016 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+from .. import _yaml
+
+
+class MetaElement():
+
+    # MetaElement()
+    #
+    # An abstract object holding data suitable for constructing an Element
+    #
+    # Args:
+    #    project: The project that contains the element
+    #    name: The resolved element name
+    #    kind: The element kind
+    #    provenance: The provenance of the element
+    #    sources: An array of MetaSource objects
+    #    config: The configuration data for the element
+    #    variables: The variables declared or overridden on this element
+    #    environment: The environment variables declared or overridden on this element
+    #    env_nocache: List of environment vars which should not be considered in cache keys
+    #    public: Public domain data dictionary
+    #    sandbox: Configuration specific to the sandbox environment
+    #    first_pass: The element is to be loaded with first pass configuration (junction)
+    #
+    def __init__(self, project, name, kind=None, provenance=None, sources=None, config=None,
+                 variables=None, environment=None, env_nocache=None, public=None,
+                 sandbox=None, first_pass=False):
+        self.project = project
+        self.name = name
+        self.kind = kind
+        self.provenance = provenance
+        self.sources = sources
+        self.config = config or _yaml.new_empty_node()
+        self.variables = variables or _yaml.new_empty_node()
+        self.environment = environment or _yaml.new_empty_node()
+        self.env_nocache = env_nocache or []
+        self.public = public or _yaml.new_empty_node()
+        self.sandbox = sandbox or _yaml.new_empty_node()
+        self.build_dependencies = []
+        self.dependencies = []
+        self.first_pass = first_pass
+        self.is_junction = kind == "junction"
diff --git a/src/buildstream/_loader/metasource.py b/src/buildstream/_loader/metasource.py
new file mode 100644
index 000000000..da2c0e292
--- /dev/null
+++ b/src/buildstream/_loader/metasource.py
@@ -0,0 +1,42 @@
+#
+#  Copyright (C) 2016 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+
+class MetaSource():
+
+    # MetaSource()
+    #
+    # An abstract object holding data suitable for constructing a Source
+    #
+    # Args:
+    #    element_name: The name of the owning element
+    #    element_index: The index of the source in the owning element's source list
+    #    element_kind: The kind of the owning element
+    #    kind: The kind of the source
+    #    config: The configuration data for the source
+    #    first_pass: This source will be used with first project pass configuration (used for junctions).
+    #
+    def __init__(self, element_name, element_index, element_kind, kind, config, directory):
+        self.element_name = element_name
+        self.element_index = element_index
+        self.element_kind = element_kind
+        self.kind = kind
+        self.config = config
+        self.directory = directory
+        self.first_pass = False
diff --git a/src/buildstream/_loader/types.py b/src/buildstream/_loader/types.py
new file mode 100644
index 000000000..f9dd38ca0
--- /dev/null
+++ b/src/buildstream/_loader/types.py
@@ -0,0 +1,112 @@
+#
+#  Copyright (C) 2018 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+from .._exceptions import LoadError, LoadErrorReason
+from .. import _yaml
+
+
+# Symbol():
+#
+# A simple object to denote the symbols we load with from YAML
+#
+class Symbol():
+    FILENAME = "filename"
+    KIND = "kind"
+    DEPENDS = "depends"
+    BUILD_DEPENDS = "build-depends"
+    RUNTIME_DEPENDS = "runtime-depends"
+    SOURCES = "sources"
+    CONFIG = "config"
+    VARIABLES = "variables"
+    ENVIRONMENT = "environment"
+    ENV_NOCACHE = "environment-nocache"
+    PUBLIC = "public"
+    TYPE = "type"
+    BUILD = "build"
+    RUNTIME = "runtime"
+    ALL = "all"
+    DIRECTORY = "directory"
+    JUNCTION = "junction"
+    SANDBOX = "sandbox"
+
+
+# Dependency()
+#
+# A simple object describing a dependency
+#
+# Args:
+#    name (str): The element name
+#    dep_type (str): The type of dependency, can be
+#                    Symbol.ALL, Symbol.BUILD, or Symbol.RUNTIME
+#    junction (str): The element name of the junction, or None
+#    provenance (Provenance): The YAML node provenance of where this
+#                             dependency was declared
+#
+class Dependency():
+    def __init__(self, dep, provenance, default_dep_type=None):
+        self.provenance = provenance
+
+        if isinstance(dep, str):
+            self.name = dep
+            self.dep_type = default_dep_type
+            self.junction = None
+
+        elif _yaml.is_node(dep):
+            if default_dep_type:
+                _yaml.node_validate(dep, ['filename', 'junction'])
+                dep_type = default_dep_type
+            else:
+                _yaml.node_validate(dep, ['filename', 'type', 'junction'])
+
+                # Make type optional, for this we set it to None
+                dep_type = _yaml.node_get(dep, str, Symbol.TYPE, default_value=None)
+                if dep_type is None or dep_type == Symbol.ALL:
+                    dep_type = None
+                elif dep_type not in [Symbol.BUILD, Symbol.RUNTIME]:
+                    provenance = _yaml.node_get_provenance(dep, key=Symbol.TYPE)
+                    raise LoadError(LoadErrorReason.INVALID_DATA,
+                                    "{}: Dependency type '{}' is not 'build', 'runtime' or 'all'"
+                                    .format(provenance, dep_type))
+
+            self.name = _yaml.node_get(dep, str, Symbol.FILENAME)
+            self.dep_type = dep_type
+            self.junction = _yaml.node_get(dep, str, Symbol.JUNCTION, default_value=None)
+
+        else:
+            raise LoadError(LoadErrorReason.INVALID_DATA,
+                            "{}: Dependency is not specified as a string or a dictionary".format(provenance))
+
+        # `:` characters are not allowed in filename if a junction was
+        # explicitly specified
+        if self.junction and ':' in self.name:
+            raise LoadError(LoadErrorReason.INVALID_DATA,
+                            "{}: Dependency {} contains `:` in its name. "
+                            "`:` characters are not allowed in filename when "
+                            "junction attribute is specified.".format(self.provenance, self.name))
+
+        # Name of the element should never contain more than one `:` characters
+        if self.name.count(':') > 1:
+            raise LoadError(LoadErrorReason.INVALID_DATA,
+                            "{}: Dependency {} contains multiple `:` in its name. "
+                            "Recursive lookups for cross-junction elements is not "
+                            "allowed.".format(self.provenance, self.name))
+
+        # Attempt to split name if no junction was specified explicitly
+        if not self.junction and self.name.count(':') == 1:
+            self.junction, self.name = self.name.split(':')