Merge branch 'willsalmon/outOfSourecBuild' into 'master'

Out of source builds

Closes #512

See merge request BuildStream/buildstream!776

21 files changed, 195 insertions, 28 deletions

diff --git a/NEWS b/NEWS
index acbdf2aa3..7e88f4af4 100644
--- a/NEWS
+++ b/NEWS
@@ -27,6 +27,9 @@ buildstream 1.3.1
   o Generate Docker images from built artifacts using
     `contrib/bst-docker-import` script.
 
+  o Added Documentation on how to create out of source builds. This includes the
+    new the `conf-root` variable to make the process easier. And there has been
+    a bug fix to workspaces so they can be build in workspaces too.
 
 =================
 buildstream 1.1.5
diff --git a/buildstream/_versions.py b/buildstream/_versions.py
index 9d9d270c5..8ad2f8cb7 100644
--- a/buildstream/_versions.py
+++ b/buildstream/_versions.py
@@ -23,7 +23,7 @@
 # This version is bumped whenever enhancements are made
 # to the `project.conf` format or the core element format.
 #
-BST_FORMAT_VERSION = 16
+BST_FORMAT_VERSION = 17
 
 
 # The base BuildStream artifact version
diff --git a/buildstream/buildelement.py b/buildstream/buildelement.py
index 5447c13be..e04ee38b0 100644
--- a/buildstream/buildelement.py
+++ b/buildstream/buildelement.py
@@ -23,6 +23,50 @@ BuildElement - Abstract class for build elements
 The BuildElement class is a convenience element one can derive from for
 implementing the most common case of element.
 
+Built-in functionality
+----------------------
+
+The BuildElement base class provides built in functionality that could be
+overridden by the individual plugins.
+
+This section will give a brief summary of how some of the common features work,
+some of them or the variables they use will be further detailed in the following
+sections.
+
+Location for running commands
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The ``command-subdir`` variable sets where the build commands will be executed,
+if the directory does not exist it will be created, it is defined relative to
+the buildroot.
+
+Location for configuring the project
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The ``conf-root`` is defined by default as ``.`` and is the location that
+specific build element can use to look for build configuration files. This is
+used by elements such as autotools, cmake, distutils, meson, pip and qmake.
+
+The configuration commands are run in ``command-subdir`` and by default
+``conf-root`` is ``.`` so if ``conf-root`` is not set the configuration files
+in ``command-subdir`` will be used.
+
+By setting ``conf-root`` to ``"%{build-root}/Source/conf_location"`` and your
+source elements ``directory`` variable to ``Source`` then the configuration
+files in the directory ``conf_location`` with in your Source will be used.
+The current working directory when your configuration command is run will still
+be wherever you set your ``command-subdir`` to be, regardless of where the
+configure scripts are set with ``conf-root``.
+
+.. note::
+
+   The ``conf-root`` variable is available since :ref:`format version 17 <project_format_version>`
+
+Install Location
+~~~~~~~~~~~~~~~~
+
+You should not change the ``install-root`` variable as it is a special
+writeable location in the sandbox but it is useful when writing custom
+install instructions as it may need to be supplied as the ``DESTDIR``, please
+see the :mod:`cmake <elements.cmake>` build element for example.
 
 Abstract method implementations
 -------------------------------
diff --git a/buildstream/data/projectconfig.yaml b/buildstream/data/projectconfig.yaml
index 4d2ccc647..bc9e5147d 100644
--- a/buildstream/data/projectconfig.yaml
+++ b/buildstream/data/projectconfig.yaml
@@ -38,6 +38,9 @@ variables:
   # normally staged
   build-root: /buildstream/%{project-name}/%{element-name}
 
+  # Indicates where the build system should look for configuration files  
+  conf-root: .
+  
   # Indicates the build installation directory in the sandbox
   install-root: /buildstream-install
 
diff --git a/buildstream/plugins/elements/autotools.yaml b/buildstream/plugins/elements/autotools.yaml
index 7adafd472..a6917f869 100644
--- a/buildstream/plugins/elements/autotools.yaml
+++ b/buildstream/plugins/elements/autotools.yaml
@@ -6,11 +6,11 @@ variables:
     export NOCONFIGURE=1;
 
     if [ -x %{conf-cmd} ]; then true;
-    elif [ -x autogen ]; then ./autogen;
-    elif [ -x autogen.sh ]; then ./autogen.sh;
-    elif [ -x bootstrap ]; then ./bootstrap;
-    elif [ -x bootstrap.sh ]; then ./bootstrap.sh;
-    else autoreconf -ivf;
+    elif [ -x %{conf-root}/autogen ]; then %{conf-root}/autogen;
+    elif [ -x %{conf-root}/autogen.sh ]; then %{conf-root}/autogen.sh;
+    elif [ -x %{conf-root}/bootstrap ]; then %{conf-root}/bootstrap;
+    elif [ -x %{conf-root}/bootstrap.sh ]; then %{conf-root}/bootstrap.sh;
+    else autoreconf -ivf %{conf-root};
     fi
 
   # Project-wide extra arguments to be passed to `configure`
@@ -22,7 +22,8 @@ variables:
   # For backwards compatibility only, do not use.
   conf-extra: ''
 
-  conf-cmd: ./configure
+  conf-cmd: "%{conf-root}/configure"
+  
   conf-args: |
 
     --prefix=%{prefix} \
diff --git a/buildstream/plugins/elements/cmake.yaml b/buildstream/plugins/elements/cmake.yaml
index b51727b04..d38a2d226 100644
--- a/buildstream/plugins/elements/cmake.yaml
+++ b/buildstream/plugins/elements/cmake.yaml
@@ -23,7 +23,7 @@ variables:
 
   cmake: |
 
-    cmake -B%{build-dir} -H. -G"%{generator}" %{cmake-args}
+    cmake -B%{build-dir} -H"%{conf-root}" -G"%{generator}" %{cmake-args}
 
   make: cmake --build %{build-dir} -- ${JOBS}
   make-install: env DESTDIR="%{install-root}" cmake --build %{build-dir} --target install
diff --git a/buildstream/plugins/elements/distutils.yaml b/buildstream/plugins/elements/distutils.yaml
index 7cb6f3a8d..cec7da6e9 100644
--- a/buildstream/plugins/elements/distutils.yaml
+++ b/buildstream/plugins/elements/distutils.yaml
@@ -8,7 +8,7 @@ variables:
 
   python-build: |
 
-    %{python} setup.py build
+    %{python} %{conf-root}/setup.py build
 
   install-args: |
 
@@ -17,7 +17,7 @@ variables:
 
   python-install: |
 
-    %{python} setup.py install %{install-args}
+    %{python} %{conf-root}/setup.py install %{install-args}
 
 
 config:
diff --git a/buildstream/plugins/elements/meson.yaml b/buildstream/plugins/elements/meson.yaml
index 7af9a76e4..9636d76e7 100644
--- a/buildstream/plugins/elements/meson.yaml
+++ b/buildstream/plugins/elements/meson.yaml
@@ -28,7 +28,7 @@ variables:
     --mandir=%{mandir} \
     --infodir=%{infodir} %{meson-extra} %{meson-global} %{meson-local}
 
-  meson: meson %{build-dir} %{meson-args}
+  meson: meson %{conf-root} %{build-dir} %{meson-args}
 
   ninja: |
     ninja -j ${NINJAJOBS} -C %{build-dir}
diff --git a/buildstream/plugins/elements/pip.yaml b/buildstream/plugins/elements/pip.yaml
index 19a226e00..b2b3d3857 100644
--- a/buildstream/plugins/elements/pip.yaml
+++ b/buildstream/plugins/elements/pip.yaml
@@ -14,7 +14,7 @@ config:
   #
   install-commands:
   - |
-    %{pip} install --no-deps --root=%{install-root} --prefix=%{prefix} .
+    %{pip} install --no-deps --root=%{install-root} --prefix=%{prefix} %{conf-root} 
 
   # Commands for stripping debugging information out of
   # installed binaries
diff --git a/buildstream/plugins/elements/qmake.yaml b/buildstream/plugins/elements/qmake.yaml
index e527d45b9..38bf2dacd 100644
--- a/buildstream/plugins/elements/qmake.yaml
+++ b/buildstream/plugins/elements/qmake.yaml
@@ -2,7 +2,7 @@
 
 variables:
 
-  qmake: qmake -makefile
+  qmake: qmake -makefile %{conf-root}
   make: make
   make-install: make -j1 INSTALL_ROOT="%{install-root}" install
 
diff --git a/buildstream/sandbox/_sandboxbwrap.py b/buildstream/sandbox/_sandboxbwrap.py
index 5effadf41..8c406e53e 100644
--- a/buildstream/sandbox/_sandboxbwrap.py
+++ b/buildstream/sandbox/_sandboxbwrap.py
@@ -108,9 +108,6 @@ class SandboxBwrap(Sandbox):
             bwrap_command += ['--unshare-uts', '--hostname', 'buildstream']
             bwrap_command += ['--unshare-ipc']
 
-        if cwd is not None:
-            bwrap_command += ['--chdir', cwd]
-
         # Give it a proc and tmpfs
         bwrap_command += [
             '--proc', '/proc',
@@ -151,6 +148,10 @@ class SandboxBwrap(Sandbox):
         if flags & SandboxFlags.ROOT_READ_ONLY:
             bwrap_command += ["--remount-ro", "/"]
 
+        if cwd is not None:
+            bwrap_command += ['--dir', cwd]
+            bwrap_command += ['--chdir', cwd]
+
         # Set UID and GUI
         if self.user_ns_available:
             bwrap_command += ['--unshare-user']
@@ -179,11 +180,6 @@ class SandboxBwrap(Sandbox):
         with ExitStack() as stack:
             stack.enter_context(mount_map.mounted(self))
 
-            # Ensure the cwd exists
-            if cwd is not None:
-                workdir = os.path.join(root_mount_source, cwd.lstrip(os.sep))
-                os.makedirs(workdir, exist_ok=True)
-
             # If we're interactive, we want to inherit our stdin,
             # otherwise redirect to /dev/null, ensuring process
             # disconnected from terminal.
diff --git a/buildstream/sandbox/_sandboxchroot.py b/buildstream/sandbox/_sandboxchroot.py
index b3a2a6d9d..f19052b23 100644
--- a/buildstream/sandbox/_sandboxchroot.py
+++ b/buildstream/sandbox/_sandboxchroot.py
@@ -100,9 +100,8 @@ class SandboxChroot(Sandbox):
 
             # Ensure the cwd exists
             if cwd is not None:
-                workdir = os.path.join(root_mount_source, cwd.lstrip(os.sep))
+                workdir = os.path.join(rootfs, cwd.lstrip(os.sep))
                 os.makedirs(workdir, exist_ok=True)
-
             status = self.chroot(rootfs, command, stdin, stdout,
                                  stderr, cwd, env, flags)
 
diff --git a/buildstream/sandbox/sandbox.py b/buildstream/sandbox/sandbox.py
index 42cfb9a15..83714efdd 100644
--- a/buildstream/sandbox/sandbox.py
+++ b/buildstream/sandbox/sandbox.py
@@ -223,7 +223,9 @@ class Sandbox():
         .. note::
 
            The optional *cwd* argument will default to the value set with
-           :func:`~buildstream.sandbox.Sandbox.set_work_directory`
+           :func:`~buildstream.sandbox.Sandbox.set_work_directory` and this
+           function must make sure the directory will be created if it does
+           not exist yet, even if a workspace is being used.
         """
         raise ImplError("Sandbox of type '{}' does not implement run()"
                         .format(type(self).__name__))
diff --git a/buildstream/source.py b/buildstream/source.py
index 30883430a..ad7c57fb0 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -20,6 +20,19 @@
 Source - Base source class
 ==========================
 
+Built-in functionality
+----------------------
+
+The Source base class provides built in functionality that may be overridden
+by individual plugins.
+
+* Directory
+
+  The ``directory`` variable can be set for all sources of a type in project.conf
+  or per source within a element.
+
+  This sets the location within the build root that the content of the source
+  will be loaded in to. If the location does not exist, it will be created.
 
 .. _core_source_abstract_methods:
 
diff --git a/tests/format/variables.py b/tests/format/variables.py
index d01d87e5b..26bb3db98 100644
--- a/tests/format/variables.py
+++ b/tests/format/variables.py
@@ -19,10 +19,10 @@ DATA_DIR = os.path.join(
 @pytest.mark.parametrize("target,varname,expected", [
     ('autotools.bst', 'make-install', "make -j1 DESTDIR=\"/buildstream-install\" install"),
     ('cmake.bst', 'cmake',
-     "cmake -B_builddir -H. -G\"Unix Makefiles\" -DCMAKE_INSTALL_PREFIX:PATH=\"/usr\" \\\n" +
+     "cmake -B_builddir -H\".\" -G\"Unix Makefiles\" " + "-DCMAKE_INSTALL_PREFIX:PATH=\"/usr\" \\\n" +
      "-DCMAKE_INSTALL_LIBDIR=lib   "),
     ('distutils.bst', 'python-install',
-     "python3 setup.py install --prefix \"/usr\" \\\n" +
+     "python3 ./setup.py install --prefix \"/usr\" \\\n" +
      "--root \"/buildstream-install\""),
     ('makemaker.bst', 'configure', "perl Makefile.PL PREFIX=/buildstream-install/usr"),
     ('modulebuild.bst', 'configure', "perl Build.PL --prefix \"/buildstream-install/usr\""),
@@ -45,10 +45,10 @@ def test_defaults(cli, datafiles, tmpdir, target, varname, expected):
 @pytest.mark.parametrize("target,varname,expected", [
     ('autotools.bst', 'make-install', "make -j1 DESTDIR=\"/custom/install/root\" install"),
     ('cmake.bst', 'cmake',
-     "cmake -B_builddir -H. -G\"Ninja\" -DCMAKE_INSTALL_PREFIX:PATH=\"/opt\" \\\n" +
+     "cmake -B_builddir -H\".\" -G\"Ninja\" " + "-DCMAKE_INSTALL_PREFIX:PATH=\"/opt\" \\\n" +
      "-DCMAKE_INSTALL_LIBDIR=lib   "),
     ('distutils.bst', 'python-install',
-     "python3 setup.py install --prefix \"/opt\" \\\n" +
+     "python3 ./setup.py install --prefix \"/opt\" \\\n" +
      "--root \"/custom/install/root\""),
     ('makemaker.bst', 'configure', "perl Makefile.PL PREFIX=/custom/install/root/opt"),
     ('modulebuild.bst', 'configure', "perl Build.PL --prefix \"/custom/install/root/opt\""),
diff --git a/tests/integration/autotools.py b/tests/integration/autotools.py
index 6ea2b667c..3c4981365 100644
--- a/tests/integration/autotools.py
+++ b/tests/integration/autotools.py
@@ -38,6 +38,30 @@ def test_autotools_build(cli, tmpdir, datafiles):
                                '/usr/share/doc/amhello/README'])
 
 
+# Test that an autotools build 'works' - we use the autotools sample
+# amhello project for this.
+@pytest.mark.integration
+@pytest.mark.datafiles(DATA_DIR)
+def test_autotools_confroot_build(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    checkout = os.path.join(cli.directory, 'checkout')
+    element_name = 'autotools/amhelloconfroot.bst'
+
+    result = cli.run(project=project, args=['build', element_name])
+    assert result.exit_code == 0
+
+    result = cli.run(project=project, args=['checkout', element_name, checkout])
+    assert result.exit_code == 0
+
+    assert_contains(checkout, ['/usr', '/usr/lib', '/usr/bin',
+                               '/usr/share', '/usr/lib/debug',
+                               '/usr/lib/debug/usr', '/usr/lib/debug/usr/bin',
+                               '/usr/lib/debug/usr/bin/hello',
+                               '/usr/bin/hello', '/usr/share/doc',
+                               '/usr/share/doc/amhello',
+                               '/usr/share/doc/amhello/README'])
+
+
 # Test running an executable built with autotools
 @pytest.mark.datafiles(DATA_DIR)
 def test_autotools_run(cli, tmpdir, datafiles):
diff --git a/tests/integration/cmake.py b/tests/integration/cmake.py
index 3c16b29b9..e74958b91 100644
--- a/tests/integration/cmake.py
+++ b/tests/integration/cmake.py
@@ -33,6 +33,24 @@ def test_cmake_build(cli, tmpdir, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
+def test_cmake_confroot_build(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    checkout = os.path.join(cli.directory, 'checkout')
+    element_name = 'cmake/cmakeconfroothello.bst'
+
+    result = cli.run(project=project, args=['build', element_name])
+    assert result.exit_code == 0
+
+    result = cli.run(project=project, args=['checkout', element_name, checkout])
+    assert result.exit_code == 0
+
+    assert_contains(checkout, ['/usr', '/usr/bin', '/usr/bin/hello',
+                               '/usr/lib/debug', '/usr/lib/debug/usr',
+                               '/usr/lib/debug/usr/bin',
+                               '/usr/lib/debug/usr/bin/hello'])
+
+
+@pytest.mark.datafiles(DATA_DIR)
 def test_cmake_run(cli, tmpdir, datafiles):
     project = os.path.join(datafiles.dirname, datafiles.basename)
     element_name = 'cmake/cmakehello.bst'
diff --git a/tests/integration/project/elements/autotools/amhelloconfroot.bst b/tests/integration/project/elements/autotools/amhelloconfroot.bst
new file mode 100644
index 000000000..28926446b
--- /dev/null
+++ b/tests/integration/project/elements/autotools/amhelloconfroot.bst
@@ -0,0 +1,15 @@
+kind: autotools
+description: Autotools test
+
+depends:
+- base.bst
+
+sources:
+- kind: tar
+  url: project_dir:/files/amhello.tar.gz
+  ref: 9ba123fa4e660929e9a0aa99f0c487b7eee59c5e7594f3284d015640b90f5590
+  directory: SourceFile
+
+variables:
+  conf-root: "%{build-root}/SourceFile"
+  command-subdir: build
diff --git a/tests/integration/project/elements/cmake/cmakeconfroothello.bst b/tests/integration/project/elements/cmake/cmakeconfroothello.bst
new file mode 100644
index 000000000..cd33dee99
--- /dev/null
+++ b/tests/integration/project/elements/cmake/cmakeconfroothello.bst
@@ -0,0 +1,15 @@
+kind: cmake
+description: Cmake test
+
+depends:
+  - base.bst
+
+sources:
+  - kind: tar
+    directory: Source
+    url: project_dir:/files/cmakehello.tar.gz
+    ref: 508266f40dbc5875293bd24c4e50a9eb6b88cbacab742033f7b92f8c087b64e5
+
+variables:
+  conf-root: "%{build-root}/Source"
+  command-subdir: build
diff --git a/tests/integration/project/elements/workspace/workspace-commanddir.bst b/tests/integration/project/elements/workspace/workspace-commanddir.bst
new file mode 100644
index 000000000..d963346d7
--- /dev/null
+++ b/tests/integration/project/elements/workspace/workspace-commanddir.bst
@@ -0,0 +1,17 @@
+kind: manual
+description: Workspace mount test
+
+depends:
+  - filename: base.bst
+    type: build
+
+sources:
+  - kind: local
+    path: files/workspace-mount-src/
+
+variables:
+  command-subdir: build
+
+config:
+  build-commands:
+    - cc -c ../hello.c
diff --git a/tests/integration/workspace.py b/tests/integration/workspace.py
index 102d053fc..bcbcd674b 100644
--- a/tests/integration/workspace.py
+++ b/tests/integration/workspace.py
@@ -34,6 +34,23 @@ def test_workspace_mount(cli, tmpdir, datafiles):
 
 @pytest.mark.integration
 @pytest.mark.datafiles(DATA_DIR)
+def test_workspace_commanddir(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    workspace = os.path.join(cli.directory, 'workspace')
+    element_name = 'workspace/workspace-commanddir.bst'
+
+    res = cli.run(project=project, args=['workspace', 'open', element_name, workspace])
+    assert res.exit_code == 0
+
+    res = cli.run(project=project, args=['build', element_name])
+    assert res.exit_code == 0
+
+    assert os.path.exists(os.path.join(cli.directory, 'workspace'))
+    assert os.path.exists(os.path.join(cli.directory, 'workspace', 'build'))
+
+
+@pytest.mark.integration
+@pytest.mark.datafiles(DATA_DIR)
 def test_workspace_updated_dependency(cli, tmpdir, datafiles):
     project = os.path.join(datafiles.dirname, datafiles.basename)
     workspace = os.path.join(cli.directory, 'workspace')