diff options
author | Phil Dawson <phildawson.0807@gmail.com> | 2018-10-16 13:19:56 +0000 |
---|---|---|
committer | Phil Dawson <phildawson.0807@gmail.com> | 2018-10-16 13:19:56 +0000 |
commit | 0a5db2290eddf7197cb499c060836ae79d27fe53 (patch) | |
tree | 408124e031f8584516d77d5b924da729da8758d1 | |
parent | 086c47c44b35c1041d39fafa4fed455ffd165433 (diff) | |
parent | 118704a9f0233ca28407db8b2bfbad2673bf3521 (diff) | |
download | buildstream-0a5db2290eddf7197cb499c060836ae79d27fe53.tar.gz |
Merge branch 'willsalmon/outOfSourecBuild' into 'master'
Out of source builds
Closes #512
See merge request BuildStream/buildstream!776
21 files changed, 195 insertions, 28 deletions
@@ -27,6 +27,9 @@ buildstream 1.3.1 o Generate Docker images from built artifacts using `contrib/bst-docker-import` script. + o Added Documentation on how to create out of source builds. This includes the + new the `conf-root` variable to make the process easier. And there has been + a bug fix to workspaces so they can be build in workspaces too. ================= buildstream 1.1.5 diff --git a/buildstream/_versions.py b/buildstream/_versions.py index 9d9d270c5..8ad2f8cb7 100644 --- a/buildstream/_versions.py +++ b/buildstream/_versions.py @@ -23,7 +23,7 @@ # This version is bumped whenever enhancements are made # to the `project.conf` format or the core element format. # -BST_FORMAT_VERSION = 16 +BST_FORMAT_VERSION = 17 # The base BuildStream artifact version diff --git a/buildstream/buildelement.py b/buildstream/buildelement.py index 5447c13be..e04ee38b0 100644 --- a/buildstream/buildelement.py +++ b/buildstream/buildelement.py @@ -23,6 +23,50 @@ BuildElement - Abstract class for build elements The BuildElement class is a convenience element one can derive from for implementing the most common case of element. +Built-in functionality +---------------------- + +The BuildElement base class provides built in functionality that could be +overridden by the individual plugins. + +This section will give a brief summary of how some of the common features work, +some of them or the variables they use will be further detailed in the following +sections. + +Location for running commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The ``command-subdir`` variable sets where the build commands will be executed, +if the directory does not exist it will be created, it is defined relative to +the buildroot. + +Location for configuring the project +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The ``conf-root`` is defined by default as ``.`` and is the location that +specific build element can use to look for build configuration files. This is +used by elements such as autotools, cmake, distutils, meson, pip and qmake. + +The configuration commands are run in ``command-subdir`` and by default +``conf-root`` is ``.`` so if ``conf-root`` is not set the configuration files +in ``command-subdir`` will be used. + +By setting ``conf-root`` to ``"%{build-root}/Source/conf_location"`` and your +source elements ``directory`` variable to ``Source`` then the configuration +files in the directory ``conf_location`` with in your Source will be used. +The current working directory when your configuration command is run will still +be wherever you set your ``command-subdir`` to be, regardless of where the +configure scripts are set with ``conf-root``. + +.. note:: + + The ``conf-root`` variable is available since :ref:`format version 17 <project_format_version>` + +Install Location +~~~~~~~~~~~~~~~~ + +You should not change the ``install-root`` variable as it is a special +writeable location in the sandbox but it is useful when writing custom +install instructions as it may need to be supplied as the ``DESTDIR``, please +see the :mod:`cmake <elements.cmake>` build element for example. Abstract method implementations ------------------------------- diff --git a/buildstream/data/projectconfig.yaml b/buildstream/data/projectconfig.yaml index 4d2ccc647..bc9e5147d 100644 --- a/buildstream/data/projectconfig.yaml +++ b/buildstream/data/projectconfig.yaml @@ -38,6 +38,9 @@ variables: # normally staged build-root: /buildstream/%{project-name}/%{element-name} + # Indicates where the build system should look for configuration files + conf-root: . + # Indicates the build installation directory in the sandbox install-root: /buildstream-install diff --git a/buildstream/plugins/elements/autotools.yaml b/buildstream/plugins/elements/autotools.yaml index 7adafd472..a6917f869 100644 --- a/buildstream/plugins/elements/autotools.yaml +++ b/buildstream/plugins/elements/autotools.yaml @@ -6,11 +6,11 @@ variables: export NOCONFIGURE=1; if [ -x %{conf-cmd} ]; then true; - elif [ -x autogen ]; then ./autogen; - elif [ -x autogen.sh ]; then ./autogen.sh; - elif [ -x bootstrap ]; then ./bootstrap; - elif [ -x bootstrap.sh ]; then ./bootstrap.sh; - else autoreconf -ivf; + elif [ -x %{conf-root}/autogen ]; then %{conf-root}/autogen; + elif [ -x %{conf-root}/autogen.sh ]; then %{conf-root}/autogen.sh; + elif [ -x %{conf-root}/bootstrap ]; then %{conf-root}/bootstrap; + elif [ -x %{conf-root}/bootstrap.sh ]; then %{conf-root}/bootstrap.sh; + else autoreconf -ivf %{conf-root}; fi # Project-wide extra arguments to be passed to `configure` @@ -22,7 +22,8 @@ variables: # For backwards compatibility only, do not use. conf-extra: '' - conf-cmd: ./configure + conf-cmd: "%{conf-root}/configure" + conf-args: | --prefix=%{prefix} \ diff --git a/buildstream/plugins/elements/cmake.yaml b/buildstream/plugins/elements/cmake.yaml index b51727b04..d38a2d226 100644 --- a/buildstream/plugins/elements/cmake.yaml +++ b/buildstream/plugins/elements/cmake.yaml @@ -23,7 +23,7 @@ variables: cmake: | - cmake -B%{build-dir} -H. -G"%{generator}" %{cmake-args} + cmake -B%{build-dir} -H"%{conf-root}" -G"%{generator}" %{cmake-args} make: cmake --build %{build-dir} -- ${JOBS} make-install: env DESTDIR="%{install-root}" cmake --build %{build-dir} --target install diff --git a/buildstream/plugins/elements/distutils.yaml b/buildstream/plugins/elements/distutils.yaml index 7cb6f3a8d..cec7da6e9 100644 --- a/buildstream/plugins/elements/distutils.yaml +++ b/buildstream/plugins/elements/distutils.yaml @@ -8,7 +8,7 @@ variables: python-build: | - %{python} setup.py build + %{python} %{conf-root}/setup.py build install-args: | @@ -17,7 +17,7 @@ variables: python-install: | - %{python} setup.py install %{install-args} + %{python} %{conf-root}/setup.py install %{install-args} config: diff --git a/buildstream/plugins/elements/meson.yaml b/buildstream/plugins/elements/meson.yaml index 7af9a76e4..9636d76e7 100644 --- a/buildstream/plugins/elements/meson.yaml +++ b/buildstream/plugins/elements/meson.yaml @@ -28,7 +28,7 @@ variables: --mandir=%{mandir} \ --infodir=%{infodir} %{meson-extra} %{meson-global} %{meson-local} - meson: meson %{build-dir} %{meson-args} + meson: meson %{conf-root} %{build-dir} %{meson-args} ninja: | ninja -j ${NINJAJOBS} -C %{build-dir} diff --git a/buildstream/plugins/elements/pip.yaml b/buildstream/plugins/elements/pip.yaml index 19a226e00..b2b3d3857 100644 --- a/buildstream/plugins/elements/pip.yaml +++ b/buildstream/plugins/elements/pip.yaml @@ -14,7 +14,7 @@ config: # install-commands: - | - %{pip} install --no-deps --root=%{install-root} --prefix=%{prefix} . + %{pip} install --no-deps --root=%{install-root} --prefix=%{prefix} %{conf-root} # Commands for stripping debugging information out of # installed binaries diff --git a/buildstream/plugins/elements/qmake.yaml b/buildstream/plugins/elements/qmake.yaml index e527d45b9..38bf2dacd 100644 --- a/buildstream/plugins/elements/qmake.yaml +++ b/buildstream/plugins/elements/qmake.yaml @@ -2,7 +2,7 @@ variables: - qmake: qmake -makefile + qmake: qmake -makefile %{conf-root} make: make make-install: make -j1 INSTALL_ROOT="%{install-root}" install diff --git a/buildstream/sandbox/_sandboxbwrap.py b/buildstream/sandbox/_sandboxbwrap.py index 5effadf41..8c406e53e 100644 --- a/buildstream/sandbox/_sandboxbwrap.py +++ b/buildstream/sandbox/_sandboxbwrap.py @@ -108,9 +108,6 @@ class SandboxBwrap(Sandbox): bwrap_command += ['--unshare-uts', '--hostname', 'buildstream'] bwrap_command += ['--unshare-ipc'] - if cwd is not None: - bwrap_command += ['--chdir', cwd] - # Give it a proc and tmpfs bwrap_command += [ '--proc', '/proc', @@ -151,6 +148,10 @@ class SandboxBwrap(Sandbox): if flags & SandboxFlags.ROOT_READ_ONLY: bwrap_command += ["--remount-ro", "/"] + if cwd is not None: + bwrap_command += ['--dir', cwd] + bwrap_command += ['--chdir', cwd] + # Set UID and GUI if self.user_ns_available: bwrap_command += ['--unshare-user'] @@ -179,11 +180,6 @@ class SandboxBwrap(Sandbox): with ExitStack() as stack: stack.enter_context(mount_map.mounted(self)) - # Ensure the cwd exists - if cwd is not None: - workdir = os.path.join(root_mount_source, cwd.lstrip(os.sep)) - os.makedirs(workdir, exist_ok=True) - # If we're interactive, we want to inherit our stdin, # otherwise redirect to /dev/null, ensuring process # disconnected from terminal. diff --git a/buildstream/sandbox/_sandboxchroot.py b/buildstream/sandbox/_sandboxchroot.py index b3a2a6d9d..f19052b23 100644 --- a/buildstream/sandbox/_sandboxchroot.py +++ b/buildstream/sandbox/_sandboxchroot.py @@ -100,9 +100,8 @@ class SandboxChroot(Sandbox): # Ensure the cwd exists if cwd is not None: - workdir = os.path.join(root_mount_source, cwd.lstrip(os.sep)) + workdir = os.path.join(rootfs, cwd.lstrip(os.sep)) os.makedirs(workdir, exist_ok=True) - status = self.chroot(rootfs, command, stdin, stdout, stderr, cwd, env, flags) diff --git a/buildstream/sandbox/sandbox.py b/buildstream/sandbox/sandbox.py index 42cfb9a15..83714efdd 100644 --- a/buildstream/sandbox/sandbox.py +++ b/buildstream/sandbox/sandbox.py @@ -223,7 +223,9 @@ class Sandbox(): .. note:: The optional *cwd* argument will default to the value set with - :func:`~buildstream.sandbox.Sandbox.set_work_directory` + :func:`~buildstream.sandbox.Sandbox.set_work_directory` and this + function must make sure the directory will be created if it does + not exist yet, even if a workspace is being used. """ raise ImplError("Sandbox of type '{}' does not implement run()" .format(type(self).__name__)) diff --git a/buildstream/source.py b/buildstream/source.py index 30883430a..ad7c57fb0 100644 --- a/buildstream/source.py +++ b/buildstream/source.py @@ -20,6 +20,19 @@ Source - Base source class ========================== +Built-in functionality +---------------------- + +The Source base class provides built in functionality that may be overridden +by individual plugins. + +* Directory + + The ``directory`` variable can be set for all sources of a type in project.conf + or per source within a element. + + This sets the location within the build root that the content of the source + will be loaded in to. If the location does not exist, it will be created. .. _core_source_abstract_methods: diff --git a/tests/format/variables.py b/tests/format/variables.py index d01d87e5b..26bb3db98 100644 --- a/tests/format/variables.py +++ b/tests/format/variables.py @@ -19,10 +19,10 @@ DATA_DIR = os.path.join( @pytest.mark.parametrize("target,varname,expected", [ ('autotools.bst', 'make-install', "make -j1 DESTDIR=\"/buildstream-install\" install"), ('cmake.bst', 'cmake', - "cmake -B_builddir -H. -G\"Unix Makefiles\" -DCMAKE_INSTALL_PREFIX:PATH=\"/usr\" \\\n" + + "cmake -B_builddir -H\".\" -G\"Unix Makefiles\" " + "-DCMAKE_INSTALL_PREFIX:PATH=\"/usr\" \\\n" + "-DCMAKE_INSTALL_LIBDIR=lib "), ('distutils.bst', 'python-install', - "python3 setup.py install --prefix \"/usr\" \\\n" + + "python3 ./setup.py install --prefix \"/usr\" \\\n" + "--root \"/buildstream-install\""), ('makemaker.bst', 'configure', "perl Makefile.PL PREFIX=/buildstream-install/usr"), ('modulebuild.bst', 'configure', "perl Build.PL --prefix \"/buildstream-install/usr\""), @@ -45,10 +45,10 @@ def test_defaults(cli, datafiles, tmpdir, target, varname, expected): @pytest.mark.parametrize("target,varname,expected", [ ('autotools.bst', 'make-install', "make -j1 DESTDIR=\"/custom/install/root\" install"), ('cmake.bst', 'cmake', - "cmake -B_builddir -H. -G\"Ninja\" -DCMAKE_INSTALL_PREFIX:PATH=\"/opt\" \\\n" + + "cmake -B_builddir -H\".\" -G\"Ninja\" " + "-DCMAKE_INSTALL_PREFIX:PATH=\"/opt\" \\\n" + "-DCMAKE_INSTALL_LIBDIR=lib "), ('distutils.bst', 'python-install', - "python3 setup.py install --prefix \"/opt\" \\\n" + + "python3 ./setup.py install --prefix \"/opt\" \\\n" + "--root \"/custom/install/root\""), ('makemaker.bst', 'configure', "perl Makefile.PL PREFIX=/custom/install/root/opt"), ('modulebuild.bst', 'configure', "perl Build.PL --prefix \"/custom/install/root/opt\""), diff --git a/tests/integration/autotools.py b/tests/integration/autotools.py index 6ea2b667c..3c4981365 100644 --- a/tests/integration/autotools.py +++ b/tests/integration/autotools.py @@ -38,6 +38,30 @@ def test_autotools_build(cli, tmpdir, datafiles): '/usr/share/doc/amhello/README']) +# Test that an autotools build 'works' - we use the autotools sample +# amhello project for this. +@pytest.mark.integration +@pytest.mark.datafiles(DATA_DIR) +def test_autotools_confroot_build(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + checkout = os.path.join(cli.directory, 'checkout') + element_name = 'autotools/amhelloconfroot.bst' + + result = cli.run(project=project, args=['build', element_name]) + assert result.exit_code == 0 + + result = cli.run(project=project, args=['checkout', element_name, checkout]) + assert result.exit_code == 0 + + assert_contains(checkout, ['/usr', '/usr/lib', '/usr/bin', + '/usr/share', '/usr/lib/debug', + '/usr/lib/debug/usr', '/usr/lib/debug/usr/bin', + '/usr/lib/debug/usr/bin/hello', + '/usr/bin/hello', '/usr/share/doc', + '/usr/share/doc/amhello', + '/usr/share/doc/amhello/README']) + + # Test running an executable built with autotools @pytest.mark.datafiles(DATA_DIR) def test_autotools_run(cli, tmpdir, datafiles): diff --git a/tests/integration/cmake.py b/tests/integration/cmake.py index 3c16b29b9..e74958b91 100644 --- a/tests/integration/cmake.py +++ b/tests/integration/cmake.py @@ -33,6 +33,24 @@ def test_cmake_build(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) +def test_cmake_confroot_build(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + checkout = os.path.join(cli.directory, 'checkout') + element_name = 'cmake/cmakeconfroothello.bst' + + result = cli.run(project=project, args=['build', element_name]) + assert result.exit_code == 0 + + result = cli.run(project=project, args=['checkout', element_name, checkout]) + assert result.exit_code == 0 + + assert_contains(checkout, ['/usr', '/usr/bin', '/usr/bin/hello', + '/usr/lib/debug', '/usr/lib/debug/usr', + '/usr/lib/debug/usr/bin', + '/usr/lib/debug/usr/bin/hello']) + + +@pytest.mark.datafiles(DATA_DIR) def test_cmake_run(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) element_name = 'cmake/cmakehello.bst' diff --git a/tests/integration/project/elements/autotools/amhelloconfroot.bst b/tests/integration/project/elements/autotools/amhelloconfroot.bst new file mode 100644 index 000000000..28926446b --- /dev/null +++ b/tests/integration/project/elements/autotools/amhelloconfroot.bst @@ -0,0 +1,15 @@ +kind: autotools +description: Autotools test + +depends: +- base.bst + +sources: +- kind: tar + url: project_dir:/files/amhello.tar.gz + ref: 9ba123fa4e660929e9a0aa99f0c487b7eee59c5e7594f3284d015640b90f5590 + directory: SourceFile + +variables: + conf-root: "%{build-root}/SourceFile" + command-subdir: build diff --git a/tests/integration/project/elements/cmake/cmakeconfroothello.bst b/tests/integration/project/elements/cmake/cmakeconfroothello.bst new file mode 100644 index 000000000..cd33dee99 --- /dev/null +++ b/tests/integration/project/elements/cmake/cmakeconfroothello.bst @@ -0,0 +1,15 @@ +kind: cmake +description: Cmake test + +depends: + - base.bst + +sources: + - kind: tar + directory: Source + url: project_dir:/files/cmakehello.tar.gz + ref: 508266f40dbc5875293bd24c4e50a9eb6b88cbacab742033f7b92f8c087b64e5 + +variables: + conf-root: "%{build-root}/Source" + command-subdir: build diff --git a/tests/integration/project/elements/workspace/workspace-commanddir.bst b/tests/integration/project/elements/workspace/workspace-commanddir.bst new file mode 100644 index 000000000..d963346d7 --- /dev/null +++ b/tests/integration/project/elements/workspace/workspace-commanddir.bst @@ -0,0 +1,17 @@ +kind: manual +description: Workspace mount test + +depends: + - filename: base.bst + type: build + +sources: + - kind: local + path: files/workspace-mount-src/ + +variables: + command-subdir: build + +config: + build-commands: + - cc -c ../hello.c diff --git a/tests/integration/workspace.py b/tests/integration/workspace.py index 102d053fc..bcbcd674b 100644 --- a/tests/integration/workspace.py +++ b/tests/integration/workspace.py @@ -34,6 +34,23 @@ def test_workspace_mount(cli, tmpdir, datafiles): @pytest.mark.integration @pytest.mark.datafiles(DATA_DIR) +def test_workspace_commanddir(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + workspace = os.path.join(cli.directory, 'workspace') + element_name = 'workspace/workspace-commanddir.bst' + + res = cli.run(project=project, args=['workspace', 'open', element_name, workspace]) + assert res.exit_code == 0 + + res = cli.run(project=project, args=['build', element_name]) + assert res.exit_code == 0 + + assert os.path.exists(os.path.join(cli.directory, 'workspace')) + assert os.path.exists(os.path.join(cli.directory, 'workspace', 'build')) + + +@pytest.mark.integration +@pytest.mark.datafiles(DATA_DIR) def test_workspace_updated_dependency(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) workspace = os.path.join(cli.directory, 'workspace') |