18 files changed, 161 insertions, 92 deletions
diff --git a/buildstream/plugins/elements/autotools.py b/buildstream/plugins/elements/autotools.py
index 5f54c3953..14d04d9a3 100644
--- a/buildstream/plugins/elements/autotools.py
+++ b/buildstream/plugins/elements/autotools.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016, 2018 Codethink Limited
 #
@@ -18,8 +17,9 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
-"""Autotools build element
-
+"""
+autotools - Autotools build element
+===================================
 This is a :mod:`BuildElement <buildstream.buildelement>` implementation for
 using Autotools build scripts (also known as the `GNU Build System
 <https://en.wikipedia.org/wiki/GNU_Build_System>`_).
diff --git a/buildstream/plugins/elements/autotools.yaml b/buildstream/plugins/elements/autotools.yaml
index 97ab0664c..021d3815c 100644
--- a/buildstream/plugins/elements/autotools.yaml
+++ b/buildstream/plugins/elements/autotools.yaml
@@ -4,10 +4,12 @@ variables:
 
   autogen: |
     export NOCONFIGURE=1;
-    if [ -e autogen ]; then ./autogen;
-    elif [ -e autogen.sh ]; then ./autogen.sh;
-    elif [ -e bootstrap ]; then ./bootstrap;
-    elif [ -e bootstrap.sh ]; then ./bootstrap.sh;
+
+    if [ -x %{conf-cmd} ]; then true;
+    elif [ -x autogen ]; then ./autogen;
+    elif [ -x autogen.sh ]; then ./autogen.sh;
+    elif [ -x bootstrap ]; then ./bootstrap;
+    elif [ -x bootstrap.sh ]; then ./bootstrap.sh;
     else autoreconf -ivf;
     fi
 
diff --git a/buildstream/plugins/elements/cmake.py b/buildstream/plugins/elements/cmake.py
index 292785e81..8126a80ac 100644
--- a/buildstream/plugins/elements/cmake.py
+++ b/buildstream/plugins/elements/cmake.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016, 2018 Codethink Limited
 #
@@ -18,8 +17,9 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
-"""CMake build element
-
+"""
+cmake - CMake build element
+===========================
 This is a :mod:`BuildElement <buildstream.buildelement>` implementation for
 using the `CMake <https://cmake.org/>`_ build system.
 
diff --git a/buildstream/plugins/elements/compose.py b/buildstream/plugins/elements/compose.py
index 0e666c6e5..2a5979828 100644
--- a/buildstream/plugins/elements/compose.py
+++ b/buildstream/plugins/elements/compose.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
@@ -18,8 +17,9 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
-"""Compose element
-
+"""
+compose - Compose the output of multiple elements
+=================================================
 This element creates a selective composition of its dependencies.
 
 This is normally used at near the end of a pipeline to prepare
@@ -34,7 +34,6 @@ The default configuration and possible options are as such:
 """
 
 import os
-from buildstream import utils
 from buildstream import Element, Scope
 
 
@@ -56,6 +55,9 @@ class ComposeElement(Element):
     # added, to reduce the potential for confusion
     BST_FORBID_SOURCES = True
 
+    # This plugin has been modified to avoid the use of Sandbox.get_directory
+    BST_VIRTUAL_DIRECTORY = True
+
     def configure(self, node):
         self.node_validate(node, [
             'integrate', 'include', 'exclude', 'include-orphans'
@@ -104,7 +106,8 @@ class ComposeElement(Element):
                                                  orphans=self.include_orphans)
                     manifest.update(files)
 
-        basedir = sandbox.get_directory()
+        # Make a snapshot of all the files.
+        vbasedir = sandbox.get_virtual_directory()
         modified_files = set()
         removed_files = set()
         added_files = set()
@@ -116,38 +119,24 @@ class ComposeElement(Element):
                 if require_split:
 
                     # Make a snapshot of all the files before integration-commands are run.
-                    snapshot = {
-                        f: getmtime(os.path.join(basedir, f))
-                        for f in utils.list_relative_paths(basedir)
-                    }
+                    snapshot = set(vbasedir.list_relative_paths())
+                    vbasedir.mark_unmodified()
 
                 for dep in self.dependencies(Scope.BUILD):
                     dep.integrate(sandbox)
 
                 if require_split:
-
                     # Calculate added, modified and removed files
-                    basedir_contents = set(utils.list_relative_paths(basedir))
+                    post_integration_snapshot = vbasedir.list_relative_paths()
+                    modified_files = set(vbasedir.list_modified_paths())
+                    basedir_contents = set(post_integration_snapshot)
                     for path in manifest:
-                        if path in basedir_contents:
-                            if path in snapshot:
-                                preintegration_mtime = snapshot[path]
-                                if preintegration_mtime != getmtime(os.path.join(basedir, path)):
-                                    modified_files.add(path)
-                            else:
-                                # If the path appears in the manifest but not the initial snapshot,
-                                # it may be a file staged inside a directory symlink. In this case
-                                # the path we got from the manifest won't show up in the snapshot
-                                # because utils.list_relative_paths() doesn't recurse into symlink
-                                # directories.
-                                pass
-                        elif path in snapshot:
+                        if path in snapshot and path not in basedir_contents:
                             removed_files.add(path)
 
                     for path in basedir_contents:
                         if path not in snapshot:
                             added_files.add(path)
-
                     self.info("Integration modified {}, added {} and removed {} files"
                               .format(len(modified_files), len(added_files), len(removed_files)))
 
@@ -166,8 +155,7 @@ class ComposeElement(Element):
         # instead of into a subdir. The element assemble() method should
         # support this in some way.
         #
-        installdir = os.path.join(basedir, 'buildstream', 'install')
-        os.makedirs(installdir, exist_ok=True)
+        installdir = vbasedir.descend(['buildstream', 'install'], create=True)
 
         # We already saved the manifest for created files in the integration phase,
         # now collect the rest of the manifest.
@@ -191,7 +179,7 @@ class ComposeElement(Element):
 
         with self.timed_activity("Creating composition", detail=detail, silent_nested=True):
             self.info("Composing {} files".format(len(manifest)))
-            utils.link_files(basedir, installdir, files=manifest)
+            installdir.import_files(vbasedir, files=manifest, can_link=True)
 
         # And we're done
         return os.path.join(os.sep, 'buildstream', 'install')
diff --git a/buildstream/plugins/elements/distutils.py b/buildstream/plugins/elements/distutils.py
index 948e08b62..5201013c1 100644
--- a/buildstream/plugins/elements/distutils.py
+++ b/buildstream/plugins/elements/distutils.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
@@ -18,8 +17,9 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
-"""Python Distutils element
-
+"""
+distutils - Python distutils element
+====================================
 A :mod:`BuildElement <buildstream.buildelement>` implementation for using
 python distutils
 
diff --git a/buildstream/plugins/elements/filter.py b/buildstream/plugins/elements/filter.py
index 8ce16ff9f..22fddd14f 100644
--- a/buildstream/plugins/elements/filter.py
+++ b/buildstream/plugins/elements/filter.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2018 Codethink Limited
 #
@@ -18,8 +17,9 @@
 #  Authors:
 #        Jonathan Maw <jonathan.maw@codethink.co.uk>
 
-"""Filter element
-
+"""
+filter - Extract a subset of files from another element
+=======================================================
 This filters another element by producing an output that is a subset of
 the filtered element.
 
diff --git a/buildstream/plugins/elements/import.py b/buildstream/plugins/elements/import.py
index 747455d70..11c3f9a6a 100644
--- a/buildstream/plugins/elements/import.py
+++ b/buildstream/plugins/elements/import.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
@@ -18,8 +17,9 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
-"""Import element
-
+"""
+import - Import sources directly
+================================
 Import elements produce artifacts directly from its sources
 without any kind of processing. These are typically used to
 import an SDK to build on top of or to overlay your build with
@@ -31,7 +31,6 @@ The empty configuration is as such:
 """
 
 import os
-import shutil
 from buildstream import Element, BuildElement, ElementError
 
 
@@ -68,27 +67,22 @@ class ImportElement(BuildElement):
         # Do not mount workspaces as the files are copied from outside the sandbox
         self._stage_sources_in_sandbox(sandbox, 'input', mount_workspaces=False)
 
-        rootdir = sandbox.get_directory()
-        inputdir = os.path.join(rootdir, 'input')
-        outputdir = os.path.join(rootdir, 'output')
+        rootdir = sandbox.get_virtual_directory()
+        inputdir = rootdir.descend(['input'])
+        outputdir = rootdir.descend(['output'], create=True)
 
         # The directory to grab
-        inputdir = os.path.join(inputdir, self.source.lstrip(os.sep))
-        inputdir = inputdir.rstrip(os.sep)
+        inputdir = inputdir.descend(self.source.strip(os.sep).split(os.sep))
 
         # The output target directory
-        outputdir = os.path.join(outputdir, self.target.lstrip(os.sep))
-        outputdir = outputdir.rstrip(os.sep)
-
-        # Ensure target directory parent
-        os.makedirs(os.path.dirname(outputdir), exist_ok=True)
+        outputdir = outputdir.descend(self.target.strip(os.sep).split(os.sep), create=True)
 
-        if not os.path.exists(inputdir):
+        if inputdir.is_empty():
             raise ElementError("{}: No files were found inside directory '{}'"
                                .format(self, self.source))
 
         # Move it over
-        shutil.move(inputdir, outputdir)
+        outputdir.import_files(inputdir)
 
         # And we're done
         return '/output'
diff --git a/buildstream/plugins/elements/junction.py b/buildstream/plugins/elements/junction.py
index 81fd57445..ee5ed24d5 100644
--- a/buildstream/plugins/elements/junction.py
+++ b/buildstream/plugins/elements/junction.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
@@ -18,8 +17,9 @@
 #  Authors:
 #        Jürg Billeter <juerg.billeter@codethink.co.uk>
 
-"""Junction element
-
+"""
+junction - Integrate subprojects
+================================
 This element is a link to another BuildStream project. It allows integration
 of multiple projects into a single pipeline.
 
diff --git a/buildstream/plugins/elements/make.py b/buildstream/plugins/elements/make.py
new file mode 100644
index 000000000..1f37cb412
--- /dev/null
+++ b/buildstream/plugins/elements/make.py
@@ -0,0 +1,46 @@
+#
+#  Copyright Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Ed Baunton <ebaunton1@bloomberg.net>
+
+"""
+make - Make build element
+=========================
+This is a :mod:`BuildElement <buildstream.buildelement>` implementation for
+using GNU make based build.
+
+.. note::
+
+   The ``make`` element is available since :ref:`format version 9 <project_format_version>`
+
+Here is the default configuration for the ``make`` element in full:
+
+  .. literalinclude:: ../../../buildstream/plugins/elements/make.yaml
+     :language: yaml
+"""
+
+from buildstream import BuildElement
+
+
+# Element implementation for the 'make' kind.
+class MakeElement(BuildElement):
+    pass
+
+
+# Plugin entry point
+def setup():
+    return MakeElement
diff --git a/buildstream/plugins/elements/make.yaml b/buildstream/plugins/elements/make.yaml
new file mode 100644
index 000000000..1438bb52b
--- /dev/null
+++ b/buildstream/plugins/elements/make.yaml
@@ -0,0 +1,42 @@
+# make default configurations
+
+variables:
+  make: make PREFIX="%{prefix}"
+  make-install: make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install
+
+  # Set this if the sources cannot handle parallelization.
+  #
+  # notparallel: True
+
+config:
+
+  # Commands for building the software
+  #
+  build-commands:
+  - |
+    %{make}
+
+  # Commands for installing the software into a
+  # destination folder
+  #
+  install-commands:
+  - |
+    %{make-install}
+
+  # Commands for stripping debugging information out of
+  # installed binaries
+  #
+  strip-commands:
+  - |
+    %{strip-binaries}
+
+# Use max-jobs CPUs for building and enable verbosity
+environment:
+  MAKEFLAGS: -j%{max-jobs}
+  V: 1
+
+# And dont consider MAKEFLAGS or V as something which may
+# effect build output.
+environment-nocache:
+- MAKEFLAGS
+- V
diff --git a/buildstream/plugins/elements/makemaker.py b/buildstream/plugins/elements/makemaker.py
index 94d459f93..fccfaadab 100644
--- a/buildstream/plugins/elements/makemaker.py
+++ b/buildstream/plugins/elements/makemaker.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
@@ -18,8 +17,9 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
-"""Perl MakeMaker build element
-
+"""
+makemaker - Perl MakeMaker build element
+========================================
 A :mod:`BuildElement <buildstream.buildelement>` implementation for using
 the Perl ExtUtil::MakeMaker build system
 
diff --git a/buildstream/plugins/elements/manual.py b/buildstream/plugins/elements/manual.py
index 998394b05..c7bdba95f 100644
--- a/buildstream/plugins/elements/manual.py
+++ b/buildstream/plugins/elements/manual.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
@@ -18,8 +17,9 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
-"""Manual build element
-
+"""
+manual - Manual build element
+=============================
 The most basic build element does nothing but allows users to
 add custom build commands to the array understood by the :mod:`BuildElement <buildstream.buildelement>`
 
diff --git a/buildstream/plugins/elements/meson.py b/buildstream/plugins/elements/meson.py
index 2b7b7831a..228e90ad1 100644
--- a/buildstream/plugins/elements/meson.py
+++ b/buildstream/plugins/elements/meson.py
@@ -14,8 +14,9 @@
 #  You should have received a copy of the GNU Lesser General Public
 #  License along with this library. If not, see <http://www.gnu.org/licenses/>.
 
-"""Meson build element
-
+"""
+meson - Meson build element
+===========================
 This is a :mod:`BuildElement <buildstream.buildelement>` implementation for
 using `Meson <http://mesonbuild.com/>`_ build scripts.
 
diff --git a/buildstream/plugins/elements/modulebuild.py b/buildstream/plugins/elements/modulebuild.py
index c790cafb3..5189af1a6 100644
--- a/buildstream/plugins/elements/modulebuild.py
+++ b/buildstream/plugins/elements/modulebuild.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
@@ -18,8 +17,9 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
-"""Perl Module::Build build element
-
+"""
+modulebuild - Perl Module::Build build element
+==============================================
 A :mod:`BuildElement <buildstream.buildelement>` implementation for using
 the Perl Module::Build build system
 
diff --git a/buildstream/plugins/elements/pip.py b/buildstream/plugins/elements/pip.py
index b979a6d21..e62f713a6 100644
--- a/buildstream/plugins/elements/pip.py
+++ b/buildstream/plugins/elements/pip.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Mathieu Bridon
 #
@@ -18,8 +17,9 @@
 #  Authors:
 #        Mathieu Bridon <bochecha@daitauha.fr>
 
-"""Pip build element
-
+"""
+pip - Pip build element
+=======================
 A :mod:`BuildElement <buildstream.buildelement>` implementation for installing
 Python modules with pip
 
diff --git a/buildstream/plugins/elements/qmake.py b/buildstream/plugins/elements/qmake.py
index ab5843d8b..7896692a6 100644
--- a/buildstream/plugins/elements/qmake.py
+++ b/buildstream/plugins/elements/qmake.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
@@ -18,8 +17,9 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
-"""CMake build element
-
+"""
+qmake - QMake build element
+===========================
 A :mod:`BuildElement <buildstream.buildelement>` implementation for using
 the qmake build system
 
diff --git a/buildstream/plugins/elements/script.py b/buildstream/plugins/elements/script.py
index 6778b3fac..4e422c5db 100644
--- a/buildstream/plugins/elements/script.py
+++ b/buildstream/plugins/elements/script.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2017 Codethink Limited
 #
@@ -19,8 +18,9 @@
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 #        Jonathan Maw <jonathan.maw@codethink.co.uk>
 
-"""Script element
-
+"""
+script - Run scripts to create output
+=====================================
 This element allows one to run some commands to mutate the
 input and create some output.
 
diff --git a/buildstream/plugins/elements/stack.py b/buildstream/plugins/elements/stack.py
index 45c49c514..d062b23bf 100644
--- a/buildstream/plugins/elements/stack.py
+++ b/buildstream/plugins/elements/stack.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 #
 #  Copyright (C) 2016 Codethink Limited
 #
@@ -18,13 +17,13 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
-"""Stack element
-
+"""
+stack - Symbolic Element for dependency grouping
+================================================
 Stack elements are simply a symbolic element used for representing
 a logical group of elements.
 """
 
-import os
 from buildstream import Element
 
 
@@ -52,7 +51,7 @@ class StackElement(Element):
 
         # Just create a dummy empty artifact, its existence is a statement
         # that all this stack's dependencies are built.
-        rootdir = sandbox.get_directory()
+        vrootdir = sandbox.get_virtual_directory()
 
         # XXX FIXME: This is currently needed because the artifact
         #            cache wont let us commit an empty artifact.
@@ -61,10 +60,7 @@ class StackElement(Element):
         # the actual artifact data in a subdirectory, then we
         # will be able to store some additional state in the
         # artifact cache, and we can also remove this hack.
-        outputdir = os.path.join(rootdir, 'output', 'bst')
-
-        # Ensure target directory parent
-        os.makedirs(os.path.dirname(outputdir), exist_ok=True)
+        vrootdir.descend(['output', 'bst'], create=True)
 
         # And we're done
         return '/output'