Add support for sysroot'ed dependencies in BuildElement and ScriptElementvalentindavid/sysroot_dependencies_2valentindavid/sysroot_dependencies

29 files changed, 707 insertions, 108 deletions

diff --git a/buildstream/_sysroot_dependency_loader.py b/buildstream/_sysroot_dependency_loader.py
new file mode 100644
index 000000000..c9b348ceb
--- /dev/null
+++ b/buildstream/_sysroot_dependency_loader.py
@@ -0,0 +1,260 @@
+#
+#  Copyright (C) 2019 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Valentin David <valentin.david@codethink.co.uk>
+
+import os
+import itertools
+from collections.abc import Mapping
+
+from .dependency_loader import DependencyLoader, Dependency
+from . import Element, ElementError, Scope, SandboxFlags
+
+
+# SysrootDependencyLoader():
+#
+# `SysrootDependencyLoader` implements a `DependencyLoader` to extract
+# sysroot'ed dependencies.
+class SysrootDependencyLoader(DependencyLoader):
+
+    def get_dependencies(self, node):
+        sysroots = self.node_get_member(node, list, 'sysroots', default=[])
+        dependencies = []
+
+        for sysroot in sysroots:
+            depends = self.node_get_member(sysroot, list, 'depends', default=[])
+            build_depends = self.node_get_member(sysroot, list, 'build-depends', default=[])
+            depends_iter = itertools.product(['all'], depends)
+            build_depends_iter = itertools.product(['build'], build_depends)
+            for default_type, dep in itertools.chain(depends_iter, build_depends_iter):
+                if isinstance(dep, Mapping):
+                    provenance = self.node_provenance(dep)
+                    filename = self.node_get_member(node, str, 'filename')
+                    dep_type = self.node_get_member(node, str, 'type', default=default_type)
+                    junction = self.node_get_member(node, str, 'junction', default=None)
+                    dependencies.append(Dependency(filename, dep_type=dep_type,
+                                                   junction=junction, provenance=provenance))
+                else:
+                    provenance = self.node_provenance(sysroot)
+                    dependencies.append(Dependency(dep, dep_type=default_type, provenance=provenance))
+
+        return dependencies
+
+
+# SysrootHelper():
+#
+# `SysrootHelper` should be used in element plugins that use
+# `SysrootDependencyLoader` as dependency loader. It provides
+# The implementation for staging.
+class SysrootHelper:
+
+    CONFIG_KEYS = ['sysroots']
+    __layout = []
+
+    def __init__(self, element, node):
+
+        self.__element = element
+
+        for sysroot in self.__element.node_get_member(node, list, 'sysroots', []):
+            self.__element.node_validate(sysroot, ['path', 'depends', 'build-depends'])
+            path = self.__element.node_subst_member(sysroot, 'path')
+            depends = self.__element.node_get_member(sysroot, list, 'depends', default=[])
+            build_depends = self.__element.node_get_member(sysroot, list, 'build-depends', default=[])
+            for dep in itertools.chain(depends, build_depends):
+                if isinstance(dep, Mapping):
+                    self.__element.node_validate(dep, ['filename', 'type', 'junction'])
+                    filename = self.__element.node_get_member(dep, str, 'filename')
+                    junction = self.__element.node_get_member(dep, str, 'junction', default=None)
+                else:
+                    filename = dep
+                    junction = None
+                self.layout_add(filename, path, junction=junction)
+
+    # layout_add():
+    #
+    # Adds a destination where a dependency should be staged
+    #
+    # Args:
+    #    element (str): Element name of the dependency
+    #    destination (str): Path where element will be staged
+    #    junction (str): Junction of the dependency
+    #
+    # If `junction` is None, then the dependency should be in the same
+    # project as the current element.
+    #
+    # If `junction` is ignored or `Element.IGNORE_JUNCTION`, the
+    # junction of the dependency is not checked.  This is for backward
+    # compliancy and should not be used.
+    #
+    # If `element` is None, the destination will just
+    # be marked in the sandbox.
+    def layout_add(self, element, destination, *, junction=Element.IGNORE_JUNCTION):
+        #
+        # Even if this is an empty list by default, make sure that its
+        # instance data instead of appending stuff directly onto class data.
+        #
+        if not self.__layout:
+            self.__layout = []
+        item = {'element': element,
+                'destination': destination}
+        if junction is not Element.IGNORE_JUNCTION:
+            item['junction'] = junction
+        self.__layout.append(item)
+
+    # validate():
+    #
+    # Verify that elements in layouts are dependencies.
+    #
+    # Raises:
+    #    (ElementError): When a element is not in the dependencies
+    #
+    # This method is only useful when SysrootHelper.layout_add
+    # has been called directly.
+    #
+    # This should be called in implementation of Plugin.preflight.
+    def validate(self):
+        if self.__layout:
+            # Cannot proceed if layout specifies an element that isn't part
+            # of the dependencies.
+            for item in self.__layout:
+                if not item['element']:
+                    if not self.__search(item):
+                        raise ElementError("{}: '{}' in layout not found in dependencies"
+                                           .format(self.__element, item['element']))
+
+    # stage():
+    #
+    # Stage dependencies and integrate root dependencies
+    #
+    # Args:
+    #    stage_all (bool): Whether to stage all dependencies, not just the ones mapped
+    #
+    def stage(self, sandbox, stage_all):
+
+        staged = set()
+        sysroots = {}
+
+        for item in self.__layout:
+
+            # Skip layout members which dont stage an element
+            if not item['element']:
+                continue
+
+            element = self.__search(item)
+            staged.add(element)
+            if item['destination'] not in sysroots:
+                sysroots[item['destination']] = [element]
+            else:
+                sysroots[item['destination']].append(element)
+
+        if stage_all or not self.__layout:
+            for build_dep in self.__element.dependencies(Scope.BUILD, recurse=False):
+                if build_dep in staged:
+                    continue
+                if '/' not in sysroots:
+                    sysroots['/'] = [build_dep]
+                else:
+                    sysroots['/'].append(build_dep)
+
+        for sysroot, deps in sysroots.items():
+            with self.__element.timed_activity("Staging dependencies at {}".format(sysroot), silent_nested=True):
+                if sysroot != '/':
+                    virtual_dstdir = sandbox.get_virtual_directory()
+                    virtual_dstdir.descend(sysroot.lstrip(os.sep).split(os.sep), create=True)
+                all_deps = set()
+                for dep in deps:
+                    for run_dep in dep.dependencies(Scope.RUN):
+                        all_deps.add(run_dep)
+                self.__element.stage_dependency_artifacts(sandbox, Scope.BUILD, path=sysroot, dependencies=all_deps)
+
+        with sandbox.batch(SandboxFlags.NONE):
+            for item in self.__layout:
+
+                # Skip layout members which dont stage an element
+                if not item['element']:
+                    continue
+
+                element = self.__search(item)
+
+                # Integration commands can only be run for elements staged to /
+                if item['destination'] == '/':
+                    with self.__element.timed_activity("Integrating {}".format(element.name),
+                                                       silent_nested=True):
+                        for dep in element.dependencies(Scope.RUN):
+                            element.integrate(sandbox)
+
+            if stage_all or not self.__layout:
+                for build_dep in self.__element.dependencies(Scope.BUILD, recurse=False):
+                    if build_dep in staged:
+                        continue
+
+                    with self.__element.timed_activity("Integrating {}".format(build_dep.name), silent_nested=True):
+                        for dep in build_dep.dependencies(Scope.RUN):
+                            dep.integrate(sandbox)
+
+    # has_sysroots():
+    #
+    # Tells whether any element has been mapped
+    #
+    # Returns:
+    #    (bool): Whether any element has been mapped
+    def has_sysroots(self):
+        return bool(self.__layout)
+
+    # get_unique_key():
+    #
+    # Returns a value usable for an element unique key
+    #
+    # Returns:
+    #    (dict): A dictionary that uniquely identify the mapping configuration
+    def get_unique_key(self):
+        return self.__layout
+
+    # configure_sandbox():
+    #
+    # Configure the sandbox. Mark required directories in the sandbox.
+    #
+    # Args:
+    #    extra_directories (list(str)): Extra directories to mark
+    #
+    # Because Sandbox.mark_directory should be called
+    # only once, marked directories should passed as `extra_directories`
+    # instead of being marked directly.
+    def configure_sandbox(self, sandbox, extra_directories):
+
+        directories = {directory: False for directory in extra_directories}
+
+        for item in self.__layout:
+            destination = item['destination']
+            was_artifact = directories.get(destination, False)
+            directories[destination] = item['element'] or was_artifact
+
+        for directory, artifact in directories.items():
+            # Root does not need to be marked as it is always mounted
+            # with artifact (unless explicitly marked non-artifact)
+            if directory != '/':
+                sandbox.mark_directory(directory, artifact=artifact)
+
+    #
+    # Private methods
+    #
+
+    def __search(self, item):
+        if 'junction' in item:
+            return self.__element.search(Scope.BUILD, item['element'], junction=item['junction'])
+        else:
+            return self.__element.search(Scope.BUILD, item['element'])
diff --git a/buildstream/buildelement.py b/buildstream/buildelement.py
index 6ef060f12..126c966b8 100644
--- a/buildstream/buildelement.py
+++ b/buildstream/buildelement.py
@@ -135,8 +135,10 @@ artifact collection purposes.
 """
 
 import os
-from . import Element, Scope
+
+from . import Element
 from . import SandboxFlags
+from ._sysroot_dependency_loader import SysrootDependencyLoader, SysrootHelper
 
 
 # This list is preserved because of an unfortunate situation, we
@@ -157,17 +159,20 @@ _command_steps = ['configure-commands',
 
 class BuildElement(Element):
 
+    DEPENDENCY_LOADER = SysrootDependencyLoader
+
     #############################################################
     #             Abstract Method Implementations               #
     #############################################################
     def configure(self, node):
 
         self.__commands = {}  # pylint: disable=attribute-defined-outside-init
+        self.__sysroots = SysrootHelper(self, node)  # pylint: disable=attribute-defined-outside-init
 
         # FIXME: Currently this forcefully validates configurations
         #        for all BuildElement subclasses so they are unable to
         #        extend the configuration
-        self.node_validate(node, _command_steps)
+        self.node_validate(node, _command_steps + SysrootHelper.CONFIG_KEYS)
 
         for command_name in _legacy_command_steps:
             if command_name in _command_steps:
@@ -191,6 +196,9 @@ class BuildElement(Element):
         if self.get_variable('notparallel'):
             dictionary['notparallel'] = True
 
+        if self.__sysroots.has_sysroots():
+            dictionary['sysroots'] = self.__sysroots.get_unique_key()
+
         return dictionary
 
     def configure_sandbox(self, sandbox):
@@ -198,8 +206,8 @@ class BuildElement(Element):
         install_root = self.get_variable('install-root')
 
         # Tell the sandbox to mount the build root and install root
-        sandbox.mark_directory(build_root)
-        sandbox.mark_directory(install_root)
+        self.__sysroots.configure_sandbox(sandbox, [build_root,
+                                                    install_root])
 
         # Allow running all commands in a specified subdirectory
         command_subdir = self.get_variable('command-subdir')
@@ -217,15 +225,7 @@ class BuildElement(Element):
 
     def stage(self, sandbox):
 
-        # Stage deps in the sandbox root
-        with self.timed_activity("Staging dependencies", silent_nested=True):
-            self.stage_dependency_artifacts(sandbox, Scope.BUILD)
-
-        # Run any integration commands provided by the dependencies
-        # once they are all staged and ready
-        with sandbox.batch(SandboxFlags.NONE, label="Integrating sandbox"):
-            for dep in self.dependencies(Scope.BUILD):
-                dep.integrate(sandbox)
+        self.__sysroots.stage(sandbox, True)
 
         # Stage sources in the build root
         self.stage_sources(sandbox, self.get_variable('build-root'))
diff --git a/buildstream/element.py b/buildstream/element.py
index fcf7c15eb..d28390f0d 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -677,7 +677,8 @@ class Element(Plugin):
         return link_result.combine(copy_result)
 
     def stage_dependency_artifacts(self, sandbox, scope, *, path=None,
-                                   include=None, exclude=None, orphans=True):
+                                   include=None, exclude=None, orphans=True,
+                                   dependencies=None):
         """Stage element dependencies in scope
 
         This is primarily a convenience wrapper around
@@ -692,6 +693,7 @@ class Element(Plugin):
            include (list): An optional list of domains to include files from
            exclude (list): An optional list of domains to exclude files from
            orphans (bool): Whether to include files not spoken for by split domains
+           dependencies (list): An optional list of dependencies to stage
 
         Raises:
            (:class:`.ElementError`): If any of the dependencies in `scope` have not
@@ -707,7 +709,9 @@ class Element(Plugin):
         if self.__can_build_incrementally() and workspace.last_successful:
             old_dep_keys = self.__get_artifact_metadata_dependencies(workspace.last_successful)
 
-        for dep in self.dependencies(scope):
+        if dependencies is None:
+            dependencies = self.dependencies(scope)
+        for dep in dependencies:
             # If we are workspaced, and we therefore perform an
             # incremental build, we must ensure that we update the mtimes
             # of any files created by our dependencies since the last
diff --git a/buildstream/plugins/elements/script.py b/buildstream/plugins/elements/script.py
index 6c33ecf95..657584baa 100644
--- a/buildstream/plugins/elements/script.py
+++ b/buildstream/plugins/elements/script.py
@@ -36,6 +36,7 @@ The default configuration and possible options are as such:
 """
 
 import buildstream
+from buildstream._sysroot_dependency_loader import DependencyLoader
 
 
 # Element implementation for the 'script' kind.
@@ -46,6 +47,8 @@ class ScriptElement(buildstream.ScriptElement):
     BST_VIRTUAL_DIRECTORY = True
 
     def configure(self, node):
+        super().configure(node)
+
         for n in self.node_get_member(node, list, 'layout', []):
             dst = self.node_subst_member(n, 'destination')
             elm = self.node_subst_member(n, 'element', None)
@@ -53,7 +56,7 @@ class ScriptElement(buildstream.ScriptElement):
 
         self.node_validate(node, [
             'commands', 'root-read-only', 'layout'
-        ])
+        ] + self.COMMON_CONFIG_KEYS)
 
         cmds = self.node_subst_list(node, "commands")
         self.add_commands("commands", cmds)
diff --git a/buildstream/scriptelement.py b/buildstream/scriptelement.py
index 697cd2822..703c5a938 100644
--- a/buildstream/scriptelement.py
+++ b/buildstream/scriptelement.py
@@ -35,7 +35,8 @@ implementations.
 import os
 from collections import OrderedDict
 
-from . import Element, ElementError, Scope, SandboxFlags
+from . import Element, SandboxFlags
+from ._sysroot_dependency_loader import SysrootDependencyLoader, SysrootHelper
 
 
 class ScriptElement(Element):
@@ -43,7 +44,6 @@ class ScriptElement(Element):
     __cwd = "/"
     __root_read_only = False
     __commands = None
-    __layout = []
 
     # The compose element's output is its dependencies, so
     # we must rebuild if the dependencies change even when
@@ -59,6 +59,15 @@ class ScriptElement(Element):
     # added, to reduce the potential for confusion
     BST_FORBID_SOURCES = True
 
+    COMMON_CONFIG_KEYS = SysrootHelper.CONFIG_KEYS
+
+    DEPENDENCY_LOADER = SysrootDependencyLoader
+
+    def configure(self, node):
+
+        self.__stage_all = True  # pylint: disable=attribute-defined-outside-init
+        self.__sysroots = SysrootHelper(self, node)  # pylint: disable=attribute-defined-outside-init
+
     def set_work_dir(self, work_dir=None):
         """Sets the working dir
 
@@ -134,14 +143,8 @@ class ScriptElement(Element):
            In the case that no element is specified, a read-write directory will
            be made available at the specified location.
         """
-        #
-        # Even if this is an empty list by default, make sure that its
-        # instance data instead of appending stuff directly onto class data.
-        #
-        if not self.__layout:
-            self.__layout = []
-        self.__layout.append({"element": element,
-                              "destination": destination})
+        self.__stage_all = False  # pylint: disable=attribute-defined-outside-init
+        self.__sysroots.layout_add(element, destination)
 
     def add_commands(self, group_name, command_list):
         """Adds a list of commands under the group-name.
@@ -164,32 +167,15 @@ class ScriptElement(Element):
             self.__commands = OrderedDict()
         self.__commands[group_name] = command_list
 
-    def __validate_layout(self):
-        if self.__layout:
-            # Cannot proceeed if layout is used, but none are for "/"
-            root_defined = any([(entry['destination'] == '/') for entry in self.__layout])
-            if not root_defined:
-                raise ElementError("{}: Using layout, but none are staged as '/'"
-                                   .format(self))
-
-            # Cannot proceed if layout specifies an element that isn't part
-            # of the dependencies.
-            for item in self.__layout:
-                if item['element']:
-                    if not self.search(Scope.BUILD, item['element']):
-                        raise ElementError("{}: '{}' in layout not found in dependencies"
-                                           .format(self, item['element']))
-
     def preflight(self):
-        # The layout, if set, must make sense.
-        self.__validate_layout()
+        self.__sysroots.validate()
 
     def get_unique_key(self):
         return {
             'commands': self.__commands,
             'cwd': self.__cwd,
             'install-root': self.__install_root,
-            'layout': self.__layout,
+            'layout': self.__sysroots.get_unique_key(),
             'root-read-only': self.__root_read_only
         }
 
@@ -201,72 +187,11 @@ class ScriptElement(Element):
         # Setup environment
         sandbox.set_environment(self.get_environment())
 
-        # Tell the sandbox to mount the install root
-        directories = {self.__install_root: False}
-
-        # Mark the artifact directories in the layout
-        for item in self.__layout:
-            destination = item['destination']
-            was_artifact = directories.get(destination, False)
-            directories[destination] = item['element'] or was_artifact
-
-        for directory, artifact in directories.items():
-            # Root does not need to be marked as it is always mounted
-            # with artifact (unless explicitly marked non-artifact)
-            if directory != '/':
-                sandbox.mark_directory(directory, artifact=artifact)
+        self.__sysroots.configure_sandbox(sandbox, [self.__install_root])
 
     def stage(self, sandbox):
 
-        # Stage the elements, and run integration commands where appropriate.
-        if not self.__layout:
-            # if no layout set, stage all dependencies into /
-            for build_dep in self.dependencies(Scope.BUILD, recurse=False):
-                with self.timed_activity("Staging {} at /"
-                                         .format(build_dep.name), silent_nested=True):
-                    build_dep.stage_dependency_artifacts(sandbox, Scope.RUN, path="/")
-
-            with sandbox.batch(SandboxFlags.NONE):
-                for build_dep in self.dependencies(Scope.BUILD, recurse=False):
-                    with self.timed_activity("Integrating {}".format(build_dep.name), silent_nested=True):
-                        for dep in build_dep.dependencies(Scope.RUN):
-                            dep.integrate(sandbox)
-        else:
-            # If layout, follow its rules.
-            for item in self.__layout:
-
-                # Skip layout members which dont stage an element
-                if not item['element']:
-                    continue
-
-                element = self.search(Scope.BUILD, item['element'])
-                if item['destination'] == '/':
-                    with self.timed_activity("Staging {} at /".format(element.name),
-                                             silent_nested=True):
-                        element.stage_dependency_artifacts(sandbox, Scope.RUN)
-                else:
-                    with self.timed_activity("Staging {} at {}"
-                                             .format(element.name, item['destination']),
-                                             silent_nested=True):
-                        virtual_dstdir = sandbox.get_virtual_directory()
-                        virtual_dstdir.descend(item['destination'].lstrip(os.sep).split(os.sep), create=True)
-                        element.stage_dependency_artifacts(sandbox, Scope.RUN, path=item['destination'])
-
-            with sandbox.batch(SandboxFlags.NONE):
-                for item in self.__layout:
-
-                    # Skip layout members which dont stage an element
-                    if not item['element']:
-                        continue
-
-                    element = self.search(Scope.BUILD, item['element'])
-
-                    # Integration commands can only be run for elements staged to /
-                    if item['destination'] == '/':
-                        with self.timed_activity("Integrating {}".format(element.name),
-                                                 silent_nested=True):
-                            for dep in element.dependencies(Scope.RUN):
-                                dep.integrate(sandbox)
+        self.__sysroots.stage(sandbox, self.__stage_all)
 
         install_root_path_components = self.__install_root.lstrip(os.sep).split(os.sep)
         sandbox.get_virtual_directory().descend(install_root_path_components, create=True)
diff --git a/doc/source/format_declaring.rst b/doc/source/format_declaring.rst
index 714e1fa33..13858f783 100644
--- a/doc/source/format_declaring.rst
+++ b/doc/source/format_declaring.rst
@@ -159,6 +159,64 @@ See :ref:`format_dependencies` for more information on the dependency model.
 
    The ``runtime-depends`` configuration is available since :ref:`format version 14 <project_format_version>`
 
+Sysroot'ed dependencies
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Build elements and script elements can support sysroot'ed
+dependencies.  Sysroot'ed dependencies are intended for bootstraping
+base systems or cross-compiling.
+
+Because sysroot'ed dependencies are plugin specific, they are defined
+within the plugin configuration node.
+
+.. code:: yaml
+
+   config:
+     # Specify some sysroot'ed dependencies
+     sysroots:
+     - path: /sysroot
+       depends:
+       - element1.bst
+       - element2.bst
+
+During build, or initialization of build shell, sysroot'ed build
+dependencies will be staged in the given sysroot path instead of '/'
+together with the runtime dependencies of those sysroot'ed build
+dependencies.
+
+It is possible to end up with indirect runtime dependencies in
+different sysroots if they are staged from build dependencies with
+different sysroots. They will be staged multiple times.
+
+Sysroot paths only apply to build dependencies. It is not possible to
+define runtime dependencies either with ``type: runtime`` or
+``runtime-depends``. It is possible to use ``all`` dependencies, but
+the sysroot part is only for the build part not the runtime.
+
+For example:
+
+.. code:: yaml
+
+   config:
+     sysroots:
+     - path: /sysroot
+       depends:
+       - element.bst
+
+is equivalent to:
+
+.. code:: yaml
+
+   config:
+     runtime-depends:
+     - element.bst
+     sysroots:
+     - path: /sysroot
+       build-depends:
+       - element.bst
+
+:ref:`Integration commands <public_integration>` are never executed for
+sysroot'ed dependencies.
 
 .. _format_sources:
 
diff --git a/tests/sysroot_depends/project/elements/a.bst b/tests/sysroot_depends/project/elements/a.bst
new file mode 100644
index 000000000..600aed243
--- /dev/null
+++ b/tests/sysroot_depends/project/elements/a.bst
@@ -0,0 +1,4 @@
+kind: import
+sources:
+  - kind: local
+    path: files/a
diff --git a/tests/sysroot_depends/project/elements/b.bst b/tests/sysroot_depends/project/elements/b.bst
new file mode 100644
index 000000000..ebebf1150
--- /dev/null
+++ b/tests/sysroot_depends/project/elements/b.bst
@@ -0,0 +1,4 @@
+kind: import
+sources:
+  - kind: local
+    path: files/b
diff --git a/tests/sysroot_depends/project/elements/base.bst b/tests/sysroot_depends/project/elements/base.bst
new file mode 100644
index 000000000..3c38c2459
--- /dev/null
+++ b/tests/sysroot_depends/project/elements/base.bst
@@ -0,0 +1,3 @@
+kind: stack
+depends:
+- base/base-alpine.bst
diff --git a/tests/sysroot_depends/project/elements/base/base-alpine.bst b/tests/sysroot_depends/project/elements/base/base-alpine.bst
new file mode 100644
index 000000000..687588f7c
--- /dev/null
+++ b/tests/sysroot_depends/project/elements/base/base-alpine.bst
@@ -0,0 +1,12 @@
+kind: import
+
+description: |
+  Alpine Linux base for tests
+
+  Generated using the `tests/integration-tests/base/generate-base.sh` script.
+
+sources:
+  - kind: tar
+    url: alpine:integration-tests-base.v1.x86_64.tar.xz
+    base-dir: ''
+    ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639
diff --git a/tests/sysroot_depends/project/elements/compose-integration.bst b/tests/sysroot_depends/project/elements/compose-integration.bst
new file mode 100644
index 000000000..a6c5ec3f0
--- /dev/null
+++ b/tests/sysroot_depends/project/elements/compose-integration.bst
@@ -0,0 +1,6 @@
+kind: compose
+
+sysroots:
+- path: /sysroot
+  build-depends:
+  - integration.bst
diff --git a/tests/sysroot_depends/project/elements/compose-layers-with-sysroot.bst b/tests/sysroot_depends/project/elements/compose-layers-with-sysroot.bst
new file mode 100644
index 000000000..8de79508e
--- /dev/null
+++ b/tests/sysroot_depends/project/elements/compose-layers-with-sysroot.bst
@@ -0,0 +1,13 @@
+kind: manual
+
+build-depends:
+- base.bst
+
+variables:
+  install-root: "/"
+
+config:
+  sysroots:
+  - path: /other-sysroot
+    build-depends:
+    - layer2.bst
diff --git a/tests/sysroot_depends/project/elements/compose-layers.bst b/tests/sysroot_depends/project/elements/compose-layers.bst
new file mode 100644
index 000000000..498e2fc7f
--- /dev/null
+++ b/tests/sysroot_depends/project/elements/compose-layers.bst
@@ -0,0 +1,4 @@
+kind: compose
+
+build-depends:
+- layer2.bst
diff --git a/tests/sysroot_depends/project/elements/integration.bst b/tests/sysroot_depends/project/elements/integration.bst
new file mode 100644
index 000000000..e2299b91b
--- /dev/null
+++ b/tests/sysroot_depends/project/elements/integration.bst
@@ -0,0 +1,13 @@
+kind: manual
+
+depends:
+- base.bst
+
+config:
+  install-commands:
+    - echo 0 >"%{install-root}/integrated.txt"
+
+public:
+  bst:
+    integration-commands:
+    - echo 1 >/integrated.txt
diff --git a/tests/sysroot_depends/project/elements/layer1-files.bst b/tests/sysroot_depends/project/elements/layer1-files.bst
new file mode 100644
index 000000000..944d800c3
--- /dev/null
+++ b/tests/sysroot_depends/project/elements/layer1-files.bst
@@ -0,0 +1,4 @@
+kind: import
+sources:
+- kind: local
+  path: files/layer1
diff --git a/tests/sysroot_depends/project/elements/layer1.bst b/tests/sysroot_depends/project/elements/layer1.bst
new file mode 100644
index 000000000..5d72f78b8
--- /dev/null
+++ b/tests/sysroot_depends/project/elements/layer1.bst
@@ -0,0 +1,4 @@
+kind: stack
+
+depends:
+- layer1-files.bst
diff --git a/tests/sysroot_depends/project/elements/layer2-files.bst b/tests/sysroot_depends/project/elements/layer2-files.bst
new file mode 100644
index 000000000..435877d8c
--- /dev/null
+++ b/tests/sysroot_depends/project/elements/layer2-files.bst
@@ -0,0 +1,4 @@
+kind: import
+sources:
+- kind: local
+  path: files/layer2
diff --git a/tests/sysroot_depends/project/elements/layer2.bst b/tests/sysroot_depends/project/elements/layer2.bst
new file mode 100644
index 000000000..19fa16617
--- /dev/null
+++ b/tests/sysroot_depends/project/elements/layer2.bst
@@ -0,0 +1,22 @@
+kind: manual
+
+depends:
+- layer2-files.bst
+
+build-depends:
+- base.bst
+
+config:
+  sysroots:
+  - path: /sysroot
+    depends:
+    - layer1.bst
+
+  install-commands:
+  - mkdir -p "%{install-root}"
+  - |
+    for file in /*; do
+      if test -f "${file}"; then
+        cp "${file}" "%{install-root}"
+      fi
+    done
diff --git a/tests/sysroot_depends/project/elements/manual-integration-runtime.bst b/tests/sysroot_depends/project/elements/manual-integration-runtime.bst
new file mode 100644
index 000000000..0abf89e6d
--- /dev/null
+++ b/tests/sysroot_depends/project/elements/manual-integration-runtime.bst
@@ -0,0 +1,14 @@
+kind: manual
+
+depends:
+- base.bst
+
+config:
+  sysroots:
+  - path: /sysroot
+    depends:
+    - integration.bst
+
+  install-commands:
+    - mkdir -p "%{install-root}"
+    - echo dummy >"%{install-root}/dummy.txt"
diff --git a/tests/sysroot_depends/project/elements/manual-integration.bst b/tests/sysroot_depends/project/elements/manual-integration.bst
new file mode 100644
index 000000000..218a7c935
--- /dev/null
+++ b/tests/sysroot_depends/project/elements/manual-integration.bst
@@ -0,0 +1,15 @@
+kind: manual
+
+build-depends:
+- base.bst
+
+config:
+  sysroots:
+  - path: /sysroot
+    build-depends:
+    - integration.bst
+
+  install-commands:
+  - mkdir -p "%{install-root}/sysroot"
+  - if test -f /sysroot/integrated.txt; then cp /sysroot/integrated.txt "%{install-root}/sysroot"; fi
+  - if test -f /integrated.txt; then cp /integrated.txt "%{install-root}"; fi
diff --git a/tests/sysroot_depends/project/elements/sysroot-integration.bst b/tests/sysroot_depends/project/elements/sysroot-integration.bst
new file mode 100644
index 000000000..0d2e440d4
--- /dev/null
+++ b/tests/sysroot_depends/project/elements/sysroot-integration.bst
@@ -0,0 +1,10 @@
+kind: manual
+
+variables:
+  install-root: "/"
+
+config:
+  sysroots:
+  - path: /sysroot
+    build-depends:
+    - integration.bst
diff --git a/tests/sysroot_depends/project/elements/target-variable.bst b/tests/sysroot_depends/project/elements/target-variable.bst
new file mode 100644
index 000000000..a4568ce20
--- /dev/null
+++ b/tests/sysroot_depends/project/elements/target-variable.bst
@@ -0,0 +1,14 @@
+kind: manual
+
+build-depends:
+- base.bst
+
+variables:
+  mydir: test
+  install-root: "/path"
+
+config:
+  sysroots:
+  - path: "/path/%{mydir}"
+    build-depends:
+    - b.bst
diff --git a/tests/sysroot_depends/project/elements/target.bst b/tests/sysroot_depends/project/elements/target.bst
new file mode 100644
index 000000000..5c215e44d
--- /dev/null
+++ b/tests/sysroot_depends/project/elements/target.bst
@@ -0,0 +1,14 @@
+kind: manual
+
+build-depends:
+- base.bst
+- a.bst
+
+variables:
+  install-root: '/'
+
+config:
+  sysroots:
+  - path: /sysroot
+    build-depends:
+    - b.bst
diff --git a/tests/sysroot_depends/project/files/a/a.txt b/tests/sysroot_depends/project/files/a/a.txt
new file mode 100644
index 000000000..9daeafb98
--- /dev/null
+++ b/tests/sysroot_depends/project/files/a/a.txt
@@ -0,0 +1 @@
+test
diff --git a/tests/sysroot_depends/project/files/b/b.txt b/tests/sysroot_depends/project/files/b/b.txt
new file mode 100644
index 000000000..9daeafb98
--- /dev/null
+++ b/tests/sysroot_depends/project/files/b/b.txt
@@ -0,0 +1 @@
+test
diff --git a/tests/sysroot_depends/project/files/layer1/1 b/tests/sysroot_depends/project/files/layer1/1
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/tests/sysroot_depends/project/files/layer1/1
@@ -0,0 +1 @@
+1
diff --git a/tests/sysroot_depends/project/files/layer2/2 b/tests/sysroot_depends/project/files/layer2/2
new file mode 100644
index 000000000..0cfbf0888
--- /dev/null
+++ b/tests/sysroot_depends/project/files/layer2/2
@@ -0,0 +1 @@
+2
diff --git a/tests/sysroot_depends/project/project.conf b/tests/sysroot_depends/project/project.conf
new file mode 100644
index 000000000..e479168d9
--- /dev/null
+++ b/tests/sysroot_depends/project/project.conf
@@ -0,0 +1,9 @@
+name: test
+element-path: elements
+aliases:
+  alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/
+options:
+  linux:
+    type: bool
+    description: Whether to expect a linux platform
+    default: True
diff --git a/tests/sysroot_depends/sysroot_depends.py b/tests/sysroot_depends/sysroot_depends.py
new file mode 100644
index 000000000..3dabecfc4
--- /dev/null
+++ b/tests/sysroot_depends/sysroot_depends.py
@@ -0,0 +1,176 @@
+import os
+import pytest
+from tests.testutils import cli_integration as cli
+from tests.testutils.site import IS_LINUX, HAVE_BWRAP
+
+
+# Project directory
+DATA_DIR = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    "project",
+)
+
+
+@pytest.mark.integration
+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
+@pytest.mark.datafiles(DATA_DIR)
+def test_sysroot_dependency_smoke_test(datafiles, cli, tmpdir):
+    "Test simple sysroot use case without integration"
+
+    project = str(datafiles)
+    checkout = os.path.join(str(tmpdir), 'checkout')
+
+    result = cli.run(project=project,
+                     args=['build', 'target.bst'])
+    result.assert_success()
+
+    result = cli.run(project=project,
+                     args=['checkout', 'target.bst', checkout])
+    result.assert_success()
+    assert os.path.exists(os.path.join(checkout, 'a.txt'))
+    assert os.path.exists(os.path.join(checkout, 'sysroot', 'b.txt'))
+
+
+@pytest.mark.integration
+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
+@pytest.mark.datafiles(DATA_DIR)
+def test_skip_integration_commands_build_element(datafiles, cli, tmpdir):
+    "Integration commands are not run on sysroots"
+
+    project = str(datafiles)
+    checkout = os.path.join(str(tmpdir), 'checkout')
+
+    result = cli.run(project=project,
+                     args=['build', 'manual-integration.bst'])
+    result.assert_success()
+
+    result = cli.run(project=project,
+                     args=['checkout', 'manual-integration.bst', checkout])
+    result.assert_success()
+
+    sysroot_integrated = os.path.join(checkout, 'sysroot', 'integrated.txt')
+    integrated = os.path.join(checkout, 'integrated.txt')
+    assert os.path.exists(sysroot_integrated)
+    with open(sysroot_integrated, 'r') as f:
+        assert f.read() == '0\n'
+    # We need to make sure that integration command has not been run on / either.
+    assert not os.path.exists(integrated)
+
+
+@pytest.mark.integration
+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
+@pytest.mark.datafiles(DATA_DIR)
+def test_sysroot_only_for_build(cli, tmpdir, datafiles):
+    project = str(datafiles)
+    checkout = os.path.join(str(tmpdir), 'checkout')
+
+    result = cli.run(project=project,
+                     args=['build', 'compose-layers.bst'])
+    result.assert_success()
+
+    result = cli.run(project=project,
+                     args=['checkout', 'compose-layers.bst', checkout])
+
+    result.assert_success()
+    assert os.path.exists(os.path.join(checkout, '1'))
+    assert os.path.exists(os.path.join(checkout, '2'))
+    assert not os.path.exists(os.path.join(checkout, 'sysroot', '1'))
+
+
+@pytest.mark.integration
+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
+@pytest.mark.datafiles(DATA_DIR)
+def test_sysroot_only_for_build_with_sysroot(cli, tmpdir, datafiles):
+    project = str(datafiles)
+    checkout = os.path.join(str(tmpdir), 'checkout')
+
+    result = cli.run(project=project,
+                     args=['build', 'compose-layers-with-sysroot.bst'])
+    result.assert_success()
+
+    result = cli.run(project=project,
+                     args=['checkout', 'compose-layers-with-sysroot.bst', checkout])
+
+    result.assert_success()
+    assert os.path.exists(os.path.join(checkout, 'other-sysroot', '1'))
+    assert os.path.exists(os.path.join(checkout, 'other-sysroot', '2'))
+    assert not os.path.exists(os.path.join(checkout, 'sysroot', '1'))
+
+
+@pytest.mark.integration
+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
+@pytest.mark.datafiles(DATA_DIR)
+def test_shell_no_sysroot(cli, tmpdir, datafiles):
+    "bst shell does not have sysroots and dependencies are integrated"
+
+    project = str(datafiles)
+
+    result = cli.run(project=project,
+                     args=['build', 'base.bst', 'manual-integration-runtime.bst'])
+    result.assert_success()
+
+    result = cli.run(project=project,
+                     args=['shell', 'manual-integration-runtime.bst', '--', 'cat', '/integrated.txt'])
+    result.assert_success()
+    assert result.output == '1\n'
+
+    result = cli.run(project=project,
+                     args=['shell', 'manual-integration-runtime.bst', '--', 'ls', '/sysroot/integrated.txt'])
+    assert result.exit_code != 0
+    assert result.output == ''
+
+
+@pytest.mark.integration
+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
+@pytest.mark.datafiles(DATA_DIR)
+def test_shell_build_sysroot(cli, tmpdir, datafiles):
+    "Build shell should stage build dependencies sysroot'ed non integrated"
+
+    project = str(datafiles)
+
+    result = cli.run(project=project,
+                     args=['build', 'base.bst', 'integration.bst'])
+    result.assert_success()
+
+    result = cli.run(project=project,
+                     args=['shell', '-b', 'manual-integration.bst', '--', 'cat', '/sysroot/integrated.txt'])
+    result.assert_success()
+    assert result.output == '0\n'
+
+
+@pytest.mark.integration
+@pytest.mark.datafiles(DATA_DIR)
+def test_show_dependencies_only_once(cli, tmpdir, datafiles):
+    """Dependencies should not show up in status several times when they
+    are staged with multiple sysroots"""
+
+    project = str(datafiles)
+
+    result = cli.run(project=project,
+                     args=['show', '--format', '%{name}', 'manual-integration.bst'])
+    result.assert_success()
+    pipeline = result.output.splitlines()
+    assert pipeline == ['base/base-alpine.bst',
+                        'base.bst',
+                        'integration.bst',
+                        'manual-integration.bst']
+
+
+@pytest.mark.integration
+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
+@pytest.mark.datafiles(DATA_DIR)
+def test_sysroot_path_subst_variable(datafiles, cli, tmpdir):
+    "Test that variables are expanded in sysroot path"
+
+    project = str(datafiles)
+    checkout = os.path.join(str(tmpdir), 'checkout')
+
+    result = cli.run(project=project,
+                     args=['build', 'target-variable.bst'])
+    result.assert_success()
+
+    result = cli.run(project=project,
+                     args=['checkout', 'target-variable.bst', checkout])
+    result.assert_success()
+
+    assert os.path.exists(os.path.join(checkout, 'test', 'b.txt'))