Add junction support for subprojects

This introduces junctions as a new kind of elements to allow
dependencies to cross project boundaries.

12 files changed, 399 insertions, 33 deletions

diff --git a/buildstream/_context.py b/buildstream/_context.py
index 317621f0a..9c91eebb2 100644
--- a/buildstream/_context.py
+++ b/buildstream/_context.py
@@ -45,7 +45,7 @@ from ._artifactcache import artifact_cache_specs_from_config_node
 #
 class Context():
 
-    def __init__(self):
+    def __init__(self, *, fetch_subprojects=False):
 
         # Filename indicating which configuration file was used, or None for the defaults
         self.config_origin = None
@@ -110,6 +110,7 @@ class Context():
         self._message_depth = deque()
         self._platform = None
         self._project_overrides = {}
+        self._fetch_subprojects = fetch_subprojects
 
     # load()
     #
diff --git a/buildstream/_exceptions.py b/buildstream/_exceptions.py
index da6a81365..a65ab471e 100644
--- a/buildstream/_exceptions.py
+++ b/buildstream/_exceptions.py
@@ -156,6 +156,9 @@ class LoadErrorReason(Enum):
     # A list composition directive did not apply to any underlying list
     TRAILING_LIST_DIRECTIVE = 10
 
+    # Conflicting junctions in subprojects
+    CONFLICTING_JUNCTION = 11
+
 
 # LoadError
 #
diff --git a/buildstream/_frontend/main.py b/buildstream/_frontend/main.py
index 997ecd155..c20a54b36 100644
--- a/buildstream/_frontend/main.py
+++ b/buildstream/_frontend/main.py
@@ -190,6 +190,7 @@ def cli(context, **kwargs):
 
     # Create the App, giving it the main arguments
     context.obj = App(dict(kwargs))
+    context.call_on_close(context.obj.cleanup)
 
 
 ##################################################################
@@ -219,7 +220,8 @@ def build(app, elements, all, track, track_save, track_all, track_except):
         track = elements
 
     app.initialize(elements, except_=track_except, rewritable=track_save,
-                   use_configured_remote_caches=True, track_elements=track)
+                   use_configured_remote_caches=True, track_elements=track,
+                   fetch_subprojects=True)
     app.print_heading()
     try:
         app.pipeline.build(app.scheduler, all, track, track_save)
@@ -262,7 +264,8 @@ def fetch(app, elements, deps, track, except_):
     """
 
     app.initialize(elements, except_=except_, rewritable=track,
-                   track_elements=elements if track else None)
+                   track_elements=elements if track else None,
+                   fetch_subprojects=True)
     try:
         dependencies = app.pipeline.deps_elements(deps)
         app.print_heading(deps=dependencies)
@@ -300,7 +303,8 @@ def track(app, elements, deps, except_):
         none:  No dependencies, just the element itself
         all:   All dependencies
     """
-    app.initialize(elements, except_=except_, rewritable=True, track_elements=elements)
+    app.initialize(elements, except_=except_, rewritable=True, track_elements=elements,
+                   fetch_subprojects=True)
     try:
         dependencies = app.pipeline.deps_elements(deps)
         app.print_heading(deps=dependencies)
@@ -338,7 +342,7 @@ def pull(app, elements, deps, remote):
         all:   All dependencies
     """
     app.initialize(elements, use_configured_remote_caches=(remote is None),
-                   add_remote_cache=remote)
+                   add_remote_cache=remote, fetch_subprojects=True)
     try:
         to_pull = app.pipeline.deps_elements(deps)
         app.pipeline.pull(app.scheduler, to_pull)
@@ -374,7 +378,7 @@ def push(app, elements, deps, remote):
         all:   All dependencies
     """
     app.initialize(elements, use_configured_remote_caches=(remote is None),
-                   add_remote_cache=remote)
+                   add_remote_cache=remote, fetch_subprojects=True)
     try:
         to_push = app.pipeline.deps_elements(deps)
         app.pipeline.push(app.scheduler, to_push)
@@ -784,7 +788,7 @@ class App():
     #
     def initialize(self, elements, except_=tuple(), rewritable=False,
                    use_configured_remote_caches=False, add_remote_cache=None,
-                   track_elements=None):
+                   track_elements=None, fetch_subprojects=False):
 
         profile_start(Topics.LOAD_PIPELINE, "_".join(t.replace(os.sep, '-') for t in elements))
 
@@ -792,7 +796,7 @@ class App():
         config = self.main_options['config']
 
         try:
-            self.context = Context()
+            self.context = Context(fetch_subprojects=fetch_subprojects)
             self.context.load(config)
         except BstError as e:
             click.echo("Error loading user configuration: {}".format(e), err=True)
@@ -1112,6 +1116,10 @@ class App():
         self.scheduler.resume_jobs()
         self.scheduler.connect_signals()
 
+    def cleanup(self):
+        if self.pipeline:
+            self.pipeline.cleanup()
+
 
 #
 # Return a value processor for partial choice matching.
diff --git a/buildstream/_frontend/widget.py b/buildstream/_frontend/widget.py
index 49b1e5240..ed09d113c 100644
--- a/buildstream/_frontend/widget.py
+++ b/buildstream/_frontend/widget.py
@@ -182,7 +182,7 @@ class ElementName(Widget):
             return ""
 
         plugin = _plugin_lookup(element_id)
-        name = plugin.name
+        name = plugin._get_full_name()
 
         # Sneak the action name in with the element name
         action_name = message.action_name
@@ -585,7 +585,7 @@ class LogLine(Widget):
 
             full_key, cache_key, dim_keys = element._get_full_display_key()
 
-            line = p.fmt_subst(line, 'name', element.name, fg='blue', bold=True)
+            line = p.fmt_subst(line, 'name', element._get_full_name(), fg='blue', bold=True)
             line = p.fmt_subst(line, 'key', cache_key, fg='yellow', dim=dim_keys)
             line = p.fmt_subst(line, 'full-key', full_key, fg='yellow', dim=dim_keys)
 
diff --git a/buildstream/_loader.py b/buildstream/_loader.py
index 66156da58..2cd143f49 100644
--- a/buildstream/_loader.py
+++ b/buildstream/_loader.py
@@ -21,13 +21,18 @@
 import os
 from functools import cmp_to_key
 from collections import Mapping, namedtuple
+import tempfile
+import shutil
 
 from ._exceptions import LoadError, LoadErrorReason
+from . import Consistency
+from ._project import Project
 from . import _yaml
 
 from ._metaelement import MetaElement
 from ._metasource import MetaSource
 from ._profile import Topics, profile_start, profile_end
+from ._platform import Platform
 
 
 #################################################
@@ -51,15 +56,17 @@ class Symbol():
     RUNTIME = "runtime"
     ALL = "all"
     DIRECTORY = "directory"
+    JUNCTION = "junction"
 
 
 # A simple dependency object
 #
 class Dependency():
     def __init__(self, name,
-                 dep_type=None, provenance=None):
+                 dep_type=None, junction=None, provenance=None):
         self.name = name
         self.dep_type = dep_type
+        self.junction = junction
         self.provenance = provenance
 
 
@@ -75,6 +82,13 @@ class LoadElement():
         self.name = filename
         self.loader = loader
 
+        if loader.project._junction:
+            # dependency is in subproject, qualify name
+            self.full_name = '{}:{}'.format(loader.project._junction.name, self.name)
+        else:
+            # dependency is in top-level project
+            self.full_name = self.name
+
         # Ensure the root node is valid
         _yaml.node_validate(self.data, [
             'kind', 'depends', 'sources',
@@ -98,7 +112,7 @@ class LoadElement():
     def depends(self, other):
 
         self.ensure_depends_cache()
-        return self.dep_cache.get(other.name) is not None
+        return self.dep_cache.get(other.full_name) is not None
 
     def ensure_depends_cache(self):
 
@@ -107,13 +121,13 @@ class LoadElement():
 
         self.dep_cache = {}
         for dep in self.deps:
-            elt = self.loader.elements[dep.name]
+            elt = self.loader.get_element_for_dep(dep)
 
             # Ensure the cache of the element we depend on
             elt.ensure_depends_cache()
 
             # We depend on this element
-            self.dep_cache[dep.name] = True
+            self.dep_cache[elt.full_name] = True
 
             # And we depend on everything this element depends on
             self.dep_cache.update(elt.dep_cache)
@@ -137,7 +151,7 @@ def extract_depends_from_node(data):
             dependency = Dependency(dep, provenance=dep_provenance)
 
         elif isinstance(dep, Mapping):
-            _yaml.node_validate(dep, ['filename', 'type'])
+            _yaml.node_validate(dep, ['filename', 'type', 'junction'])
 
             # Make type optional, for this we set it to None after
             dep_type = _yaml.node_get(dep, str, Symbol.TYPE, default_value="")
@@ -150,8 +164,14 @@ def extract_depends_from_node(data):
                                 .format(provenance, dep_type))
 
             filename = _yaml.node_get(dep, str, Symbol.FILENAME)
+
+            junction = _yaml.node_get(dep, str, Symbol.JUNCTION, default_value="")
+            if not junction:
+                junction = None
+
             dependency = Dependency(filename,
-                                    dep_type=dep_type, provenance=dep_provenance)
+                                    dep_type=dep_type, junction=junction,
+                                    provenance=dep_provenance)
 
         else:
             index = depends.index(dep)
@@ -178,7 +198,7 @@ def extract_depends_from_node(data):
 #
 class Loader():
 
-    def __init__(self, project, filenames):
+    def __init__(self, project, filenames, *, parent=None, tempdir=None):
 
         basedir = project.element_path
 
@@ -197,12 +217,20 @@ class Loader():
                                 .format(filename, basedir))
 
         self.project = project
+        self.context = project._context
         self.options = project._options     # Project options (OptionPool)
         self.basedir = basedir              # Base project directory
         self.targets = filenames            # Target bst elements
+        self.tempdir = tempdir
+
+        self.parent = parent
+
+        self.platform = Platform.get_platform()
+        self.artifacts = self.platform.artifactcache
 
         self.meta_elements = {}  # Dict of resolved meta elements by name
         self.elements = {}       # Dict of elements
+        self.loaders = {}        # Dict of junction loaders
 
     ########################################
     #           Main Entry Point           #
@@ -235,8 +263,8 @@ class Loader():
 
         # Set up a dummy element that depends on all top-level targets
         # to resolve potential circular dependencies between them
-        DummyTarget = namedtuple('DummyTarget', ['name', 'deps'])
-        dummy = DummyTarget(name='', deps=[Dependency(e) for e in self.targets])
+        DummyTarget = namedtuple('DummyTarget', ['name', 'full_name', 'deps'])
+        dummy = DummyTarget(name='', full_name='', deps=[Dependency(e) for e in self.targets])
         self.elements[''] = dummy
 
         profile_key = "_".join(t for t in self.targets)
@@ -256,6 +284,92 @@ class Loader():
         #
         return [self.collect_element(target) for target in self.targets]
 
+    # get_loader():
+    #
+    # Return loader for specified junction
+    #
+    # Args:
+    #    filename (str): Junction name
+    #
+    # Raises: LoadError
+    #
+    # Returns: A Loader or None if specified junction does not exist
+    def get_loader(self, filename, *, rewritable=False, ticker=None, level=0):
+        # return previously determined result
+        if filename in self.loaders:
+            loader = self.loaders[filename]
+
+            if loader is None:
+                # do not allow junctions with the same name in different
+                # subprojects
+                raise LoadError(LoadErrorReason.CONFLICTING_JUNCTION,
+                                "Conflicting junction {} in subprojects, define junction in {}"
+                                .format(filename, self.project.name))
+
+            return loader
+
+        if self.parent:
+            # junctions in the parent take precedence over junctions defined
+            # in subprojects
+            loader = self.parent.get_loader(filename, rewritable=rewritable, ticker=ticker, level=level + 1)
+            if loader:
+                self.loaders[filename] = loader
+                return loader
+
+        try:
+            load_element = self.load_file(filename, rewritable, ticker)
+        except LoadError as e:
+            if e.reason != LoadErrorReason.MISSING_FILE:
+                # other load error
+                raise
+
+            if level == 0:
+                # junction element not found in this or ancestor projects
+                raise
+            else:
+                # mark junction as not available to allow detection of
+                # conflicting junctions in subprojects
+                self.loaders[filename] = None
+                return None
+
+        # meta junction element
+        meta_element = self.collect_element(filename)
+
+        if meta_element.kind != 'junction':
+            raise LoadError(LoadErrorReason.INVALID_DATA,
+                            "{}: Expected junction but element kind is {}".format(filename, meta_element.kind))
+
+        element = meta_element.project._create_element(meta_element.kind,
+                                                       self.artifacts,
+                                                       meta_element)
+
+        os.makedirs(self.context.builddir, exist_ok=True)
+        basedir = tempfile.mkdtemp(prefix="{}-".format(element.normal_name), dir=self.context.builddir)
+
+        for meta_source in meta_element.sources:
+            source = meta_element.project._create_source(meta_source.kind,
+                                                         meta_source)
+
+            source.preflight()
+
+            if source.get_consistency() != Consistency.CACHED:
+                if self.context._fetch_subprojects:
+                    if ticker:
+                        ticker(filename, 'Fetching subproject from {} source'.format(meta_source.kind))
+                    source.fetch()
+                else:
+                    raise LoadError(LoadErrorReason.MISSING_FILE, "Subproject fetch needed for {}".format(filename))
+
+            source._stage(basedir)
+
+        project = Project(basedir, self.context, junction=element)
+
+        loader = Loader(project, [], parent=self, tempdir=basedir)
+
+        self.loaders[filename] = loader
+
+        return loader
+
     ########################################
     #             Loading Files            #
     ########################################
@@ -283,7 +397,18 @@ class Loader():
 
         # Load all dependency files for the new LoadElement
         for dep in element.deps:
-            self.load_file(dep.name, rewritable, ticker)
+            if dep.junction:
+                self.load_file(dep.junction, rewritable, ticker)
+                loader = self.get_loader(dep.junction, rewritable=rewritable, ticker=ticker)
+            else:
+                loader = self
+
+            dep_element = loader.load_file(dep.name, rewritable, ticker)
+
+            if _yaml.node_get(dep_element.data, str, Symbol.KIND) == 'junction':
+                raise LoadError(LoadErrorReason.INVALID_DATA,
+                                "{}: Cannot depend on junction"
+                                .format(dep.provenance))
 
         return element
 
@@ -303,6 +428,10 @@ class Loader():
 
         element = self.elements[element_name]
 
+        # element name must be unique across projects
+        # to be usable as key for the check_elements and validated dicts
+        element_name = element.full_name
+
         # Skip already validated branches
         if validated.get(element_name) is not None:
             return
@@ -315,7 +444,8 @@ class Loader():
         # Push / Check each dependency / Pop
         check_elements[element_name] = True
         for dep in element.deps:
-            self.check_circular_deps(dep.name, check_elements, validated)
+            loader = self.get_loader_for_dep(dep)
+            loader.check_circular_deps(dep.name, check_elements, validated)
         del check_elements[element_name]
 
         # Eliminate duplicate paths
@@ -336,16 +466,22 @@ class Loader():
         if visited is None:
             visited = {}
 
+        element = self.elements[element_name]
+
+        # element name must be unique across projects
+        # to be usable as key for the visited dict
+        element_name = element.full_name
+
         if visited.get(element_name) is not None:
             return
 
-        element = self.elements[element_name]
         for dep in element.deps:
-            self.sort_dependencies(dep.name, visited=visited)
+            loader = self.get_loader_for_dep(dep)
+            loader.sort_dependencies(dep.name, visited=visited)
 
         def dependency_cmp(dep_a, dep_b):
-            element_a = self.elements[dep_a.name]
-            element_b = self.elements[dep_b.name]
+            element_a = self.get_element_for_dep(dep_a)
+            element_b = self.get_element_for_dep(dep_b)
 
             # Sort on inter element dependency first
             if element_a.depends(element_b):
@@ -367,6 +503,18 @@ class Loader():
             elif dep_a.name < dep_b.name:
                 return -1
 
+            # Sort local elements before junction elements
+            # and use string comparison between junction elements
+            if dep_a.junction and dep_b.junction:
+                if dep_a.junction > dep_b.junction:
+                    return 1
+                elif dep_a.junction < dep_b.junction:
+                    return -1
+            elif dep_a.junction:
+                return -1
+            elif dep_b.junction:
+                return 1
+
             # This wont ever happen
             return 0
 
@@ -435,10 +583,48 @@ class Loader():
 
         # Descend
         for dep in element.deps:
-            meta_dep = self.collect_element(dep.name)
+            if kind == 'junction':
+                raise LoadError(LoadErrorReason.INVALID_DATA,
+                                "{}: Junctions do not support dependencies".format(dep.provenance))
+
+            loader = self.get_loader_for_dep(dep)
+            meta_dep = loader.collect_element(dep.name)
             if dep.dep_type != 'runtime':
                 meta_element.build_dependencies.append(meta_dep)
             if dep.dep_type != 'build':
                 meta_element.dependencies.append(meta_dep)
 
         return meta_element
+
+    def get_loader_for_dep(self, dep):
+        if dep.junction:
+            # junction dependency, delegate to appropriate loader
+            return self.loaders[dep.junction]
+        else:
+            return self
+
+    def get_element_for_dep(self, dep):
+        return self.get_loader_for_dep(dep).elements[dep.name]
+
+    # cleanup():
+    #
+    # Remove temporary checkout directories of subprojects
+    def cleanup(self):
+        if self.parent and not self.tempdir:
+            # already done
+            return
+
+        # recurse
+        for loader in self.loaders.values():
+            # value may be None with nested junctions without overrides
+            if loader is not None:
+                loader.cleanup()
+
+        if not self.parent:
+            # basedir of top-level loader is never a temporary directory
+            return
+
+        # safe guard to not accidentally delete directories outside builddir
+        if self.tempdir.startswith(self.context.builddir + os.sep):
+            if os.path.exists(self.tempdir):
+                shutil.rmtree(self.tempdir)
diff --git a/buildstream/_pipeline.py b/buildstream/_pipeline.py
index 522903f3a..fb40dd3d0 100644
--- a/buildstream/_pipeline.py
+++ b/buildstream/_pipeline.py
@@ -121,10 +121,10 @@ class Pipeline():
         self.platform = Platform.get_platform()
         self.artifacts = self.platform.artifactcache
 
-        loader = Loader(self.project, targets + except_)
+        self.loader = Loader(self.project, targets + except_)
 
         with self.timed_activity("Loading pipeline", silent_nested=True):
-            meta_elements = loader.load(rewritable, None)
+            meta_elements = self.loader.load(rewritable, None)
 
         # Resolve the real elements now that we've resolved the project
         with self.timed_activity("Resolving pipeline"):
@@ -950,3 +950,7 @@ class Pipeline():
 
             with tarfile.open(tar_name, permissions) as tar:
                 tar.add(directory, arcname=element_name)
+
+    def cleanup(self):
+        if self.loader:
+            self.loader.cleanup()
diff --git a/buildstream/_project.py b/buildstream/_project.py
index ee8b478c1..25178e12c 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -51,7 +51,7 @@ _ALIAS_SEPARATOR = ':'
 #
 class Project():
 
-    def __init__(self, directory, context, *, cli_options=None):
+    def __init__(self, directory, context, *, junction=None, cli_options=None):
 
         # The project name
         self.name = None
@@ -71,6 +71,7 @@ class Project():
         self._plugin_source_origins = []   # Origins of custom sources
         self._plugin_element_origins = []  # Origins of custom elements
         self._options = None    # Project options, the OptionPool
+        self._junction = junction   # The junction element, if this is a subproject
         self._cli_options = cli_options
         self._cache_key = None
         self._source_format_versions = {}
@@ -145,6 +146,9 @@ class Project():
         options_node = _yaml.node_get(config, Mapping, 'options', default_value={})
         self._options = OptionPool(self.element_path)
         self._options.load(options_node)
+        if self._junction:
+            # load before user configuration
+            self._options.load_yaml_values(self._junction.options, transform=self._junction._subst_string)
 
         # Collect option values specified in the user configuration
         overrides = self._context._get_overrides(self.name)
diff --git a/buildstream/element.py b/buildstream/element.py
index f7ce73288..b020fc6e1 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -216,17 +216,19 @@ class Element(Plugin):
         if visited is None:
             visited = {}
 
+        full_name = self._get_full_name()
+
         scope_set = set((Scope.BUILD, Scope.RUN)) if scope == Scope.ALL else set((scope,))
 
-        if self.name in visited and scope_set.issubset(visited[self.name]):
+        if full_name in visited and scope_set.issubset(visited[full_name]):
             return
 
         should_yield = False
-        if self.name not in visited:
-            visited[self.name] = scope_set
+        if full_name not in visited:
+            visited[full_name] = scope_set
             should_yield = True
         else:
-            visited[self.name] |= scope_set
+            visited[full_name] |= scope_set
 
         if recurse or not recursed:
             if scope == Scope.ALL:
@@ -1684,3 +1686,6 @@ class Element(Plugin):
         # Load the public data from the artifact
         metadir = os.path.join(self.__artifacts.extract(self), 'meta')
         self.__dynamic_public = _yaml.load(os.path.join(metadir, 'public.yaml'))
+
+    def _subst_string(self, value):
+        return self.__variables.subst(value)
diff --git a/buildstream/plugin.py b/buildstream/plugin.py
index cd0895f91..bfa37ef5b 100644
--- a/buildstream/plugin.py
+++ b/buildstream/plugin.py
@@ -615,6 +615,13 @@ class Plugin():
         output.flush()
         self.status('Running host command', detail=command)
 
+    def _get_full_name(self):
+        project = self.__project
+        if project._junction:
+            return '{}:{}'.format(project._junction.name, self.name)
+        else:
+            return self.name
+
 
 # Hold on to a lookup table by counter of all instantiated plugins.
 # We use this to send the id back from child processes so we can lookup
diff --git a/buildstream/plugins/elements/junction.py b/buildstream/plugins/elements/junction.py
new file mode 100644
index 000000000..c0fd67d17
--- /dev/null
+++ b/buildstream/plugins/elements/junction.py
@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+#
+#  Copyright (C) 2017 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Jürg Billeter <juerg.billeter@codethink.co.uk>
+
+"""Junction element
+
+This element is a link to another BuildStream project. It allows integration
+of multiple projects into a single pipeline.
+
+Overview
+--------
+
+.. code:: yaml
+
+   kind: junction
+
+   # Specify the BuildStream project source
+   sources:
+   - kind: git
+     url: upstream:projectname.git
+     track: master
+     ref: d0b38561afb8122a3fc6bafc5a733ec502fcaed6
+
+   # Specify the junction configuration
+   config:
+
+     # Override project options
+     options:
+       machine_arch: "%{machine_arch}"
+       debug: True
+
+.. note::
+
+   Junction elements may not specify any dependencies as they are simply
+   links to other projects and are not in the dependency graph on their own.
+
+With a junction element in place, local elements can depend on elements in
+the other BuildStream project using the additional ``junction`` attribute in the
+dependency dictionary:
+
+.. code:: yaml
+
+   depends:
+   - junction: toolchain.bst
+     filename: gcc.bst
+     type: build
+
+While junctions are elements, only a limited set of element operations is
+supported. They can be tracked and fetched like other elements.
+However, junction elements do not produce any artifacts, which means that
+they cannot be built or staged. It also means that another element cannot
+depend on a junction element itself.
+
+Sources
+-------
+``bst show`` does not implicitly fetch junction sources if they haven't been
+cached yet. However, they can be fetched explicitly:
+
+.. code::
+
+   bst fetch junction.bst
+
+Other commands such as ``bst build`` implicitly fetch junction sources.
+
+Options
+-------
+.. code:: yaml
+
+   options:
+     machine_arch: "%{machine_arch}"
+     debug: True
+
+Junctions can configure options of the linked project. Options are never
+implicitly inherited across junctions, however, variables can be used to
+explicitly assign the same value to a subproject option.
+
+Nested Junctions
+----------------
+Junctions can be nested. That is, subprojects are allowed to have junctions on
+their own. Nested junctions in different subprojects may point to the same
+project, however, in most use cases the same project should be loaded only once.
+BuildStream uses the junction element name as key to determine which junctions
+to merge. It is recommended that the name of a junction is set to the same as
+the name of the linked project.
+
+As the junctions may differ in source version and options, BuildStream cannot
+simply use one junction and ignore the others. Due to this, BuildStream requires
+the user to resolve possibly conflicting nested junctions by creating a junction
+with the same name in the top-level project, which then takes precedence.
+"""
+
+import os
+from collections import Mapping
+from buildstream import Element
+from buildstream._pipeline import PipelineError
+
+
+# Element implementation for the 'junction' kind.
+class JunctionElement(Element):
+
+    def configure(self, node):
+        self.options = self.node_get_member(node, Mapping, 'options', default={})
+
+    def preflight(self):
+        pass
+
+    def get_unique_key(self):
+        # Junctions do not produce artifacts. get_unique_key() implementation
+        # is still required for `bst fetch`.
+        return 1
+
+    def configure_sandbox(self, sandbox):
+        raise PipelineError("Cannot build junction elements")
+
+    def stage(self, sandbox):
+        raise PipelineError("Cannot stage junction elements")
+
+    def generate_script(self):
+        raise PipelineError("Cannot build junction elements")
+
+    def assemble(self, sandbox):
+        raise PipelineError("Cannot build junction elements")
+
+
+# Plugin entry point
+def setup():
+    return JunctionElement
diff --git a/doc/source/format.rst b/doc/source/format.rst
index 6f6e2f0ec..9ffe110ee 100644
--- a/doc/source/format.rst
+++ b/doc/source/format.rst
@@ -247,9 +247,13 @@ Dependency dictionary:
    depends:
    - filename: elements/foo.bst
      type: build
+     junction: elements/baseproject.bst
 
 The ``type`` attribute can be used to express the dependency type.
 
+The ``junction`` attribute can be used to depend on elements in other projects.
+See :mod:`junction <elements.junction>`.
+
 
 Dependency Types
 ~~~~~~~~~~~~~~~~
diff --git a/doc/source/pluginindex.rst b/doc/source/pluginindex.rst
index 57f14876d..0395fa0ed 100644
--- a/doc/source/pluginindex.rst
+++ b/doc/source/pluginindex.rst
@@ -20,6 +20,7 @@ General Elements
 * :mod:`import <elements.import>` - Import sources directly
 * :mod:`compose <elements.compose>` - Compose the output of multiple elements
 * :mod:`script <elements.script>` - Run scripts to create output
+* :mod:`junction <elements.junction>` - Integrate subprojects
 
 Build Elements
 ~~~~~~~~~~~~~~