doc/examples/composition: Adding user guide about split-rules and compose elementstristan/user-guide-refactor-test

17 files changed, 410 insertions, 1 deletions

diff --git a/doc/Makefile b/doc/Makefile
index a7150e6d7..4a9373d34 100644
--- a/doc/Makefile
+++ b/doc/Makefile
@@ -143,11 +143,12 @@ html devhelp: templates sessions badges
 	    $(SPHINXBUILD) -b $@ $(ALLSPHINXOPTS) "$(BUILDDIR)/$@" \
 	    $(wildcard source/*.rst) \
 	    $(wildcard source/tutorial/*.rst) \
+	    $(wildcard source/developing/*.rst) \
+	    $(wildcard source/handling-files/*.rst) \
             $(wildcard source/junctions/*.rst) \
 	    $(wildcard source/examples/*.rst) \
 	    $(wildcard source/elements/*.rst) \
 	    $(wildcard source/sources/*.rst) \
-	    $(wildcard source/developing/*.rst) \
 	    $(wildcard source/hacking/*.rst)
 	@echo
 	@echo "Build of $@ finished, output: $(CURDIR)/$(BUILDDIR)/$@"
diff --git a/doc/examples/composition/elements/base.bst b/doc/examples/composition/elements/base.bst
new file mode 100644
index 000000000..1b85a9e8c
--- /dev/null
+++ b/doc/examples/composition/elements/base.bst
@@ -0,0 +1,5 @@
+kind: stack
+description: Base stack
+
+depends:
+- base/alpine.bst
diff --git a/doc/examples/composition/elements/base/alpine.bst b/doc/examples/composition/elements/base/alpine.bst
new file mode 100644
index 000000000..be89f656f
--- /dev/null
+++ b/doc/examples/composition/elements/base/alpine.bst
@@ -0,0 +1,31 @@
+kind: import
+description: |
+
+    Alpine Linux base runtime
+
+sources:
+- kind: tar
+  url: alpine:integration-tests-base.v1.x86_64.tar.xz
+  ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639
+
+public:
+  bst:
+    #
+    # Run ldconfig in the libdir before running anything
+    #
+    integration-commands:
+    - ldconfig "%{libdir}"
+
+    #
+    # Extend the runtime split-rule domain for this element,
+    # such that we capture the runtime linker.
+    #
+    # There are various other things provided by this runtime
+    # such as tooling in /bin and an installation of python
+    # and perl, but we'll overlook these for the sake of
+    # this example.
+    #
+    split-rules:
+      runtime:
+        (>):
+        - "/lib/ld*.so*"
diff --git a/doc/examples/composition/elements/hello.bst b/doc/examples/composition/elements/hello.bst
new file mode 100644
index 000000000..c90254f10
--- /dev/null
+++ b/doc/examples/composition/elements/hello.bst
@@ -0,0 +1,22 @@
+kind: manual
+description: |
+
+  The hello application
+
+# Depend on the hello library
+depends:
+- libhello.bst
+
+# Stage the files/hello directory for building
+sources:
+  - kind: local
+    path: files/hello
+
+# Now configure the commands to run
+config:
+
+  build-commands:
+  - make PREFIX="%{prefix}"
+
+  install-commands:
+  - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install
diff --git a/doc/examples/composition/elements/libhello.bst b/doc/examples/composition/elements/libhello.bst
new file mode 100644
index 000000000..53496c84c
--- /dev/null
+++ b/doc/examples/composition/elements/libhello.bst
@@ -0,0 +1,22 @@
+kind: manual
+description: |
+
+  The libhello library
+
+# Depend on the base system
+depends:
+- base.bst
+
+# Stage the files/libhello directory for building
+sources:
+  - kind: local
+    path: files/libhello
+
+# Now configure the commands to run
+config:
+
+  build-commands:
+  - make PREFIX="%{prefix}"
+
+  install-commands:
+  - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install
diff --git a/doc/examples/composition/elements/runtime-only.bst b/doc/examples/composition/elements/runtime-only.bst
new file mode 100644
index 000000000..9885b7396
--- /dev/null
+++ b/doc/examples/composition/elements/runtime-only.bst
@@ -0,0 +1,23 @@
+kind: compose
+
+# Dependencies of a compose element cannot be transient,
+# we can only build-depend on the inputs of a composition.
+#
+build-depends:
+- hello.bst
+
+config:
+
+  # Only include files from the runtime domain
+  #
+  include:
+  - runtime
+
+  # Don't include any files which do not match any existing
+  # split rule domains.
+  #
+  include-orphans: False
+
+  # Run integration commands before composition
+  #
+  integrate: True
diff --git a/doc/examples/composition/files/hello/Makefile b/doc/examples/composition/files/hello/Makefile
new file mode 100644
index 000000000..21471d40f
--- /dev/null
+++ b/doc/examples/composition/files/hello/Makefile
@@ -0,0 +1,12 @@
+# Sample makefile for hello.c
+#
+.PHONY: all install
+
+all: hello
+
+install:
+	install -d ${DESTDIR}${PREFIX}/bin
+	install -m 755 hello ${DESTDIR}${PREFIX}/bin
+
+hello: hello.c
+	$(CC) $< -o $@ -Wall -lhello
diff --git a/doc/examples/composition/files/hello/hello.c b/doc/examples/composition/files/hello/hello.c
new file mode 100644
index 000000000..83e762c29
--- /dev/null
+++ b/doc/examples/composition/files/hello/hello.c
@@ -0,0 +1,20 @@
+/*
+ * hello.c - Simple hello program
+ */
+#include <stdio.h>
+#include <libhello.h>
+
+int main(int argc, char *argv[])
+{
+  const char *person = NULL;
+
+  if (argc > 1)
+    person = argv[1];
+
+  if (person)
+    hello(person);
+  else
+    hello("stranger");
+
+  return 0;
+}
diff --git a/doc/examples/composition/files/libhello/Makefile b/doc/examples/composition/files/libhello/Makefile
new file mode 100644
index 000000000..63ee11069
--- /dev/null
+++ b/doc/examples/composition/files/libhello/Makefile
@@ -0,0 +1,17 @@
+# Sample makefile for hello library
+#
+.PHONY: all install
+
+all: libhello.so
+
+install:
+	install -d ${DESTDIR}${PREFIX}/lib
+	install -d ${DESTDIR}${PREFIX}/include
+	install -m 644 libhello.so ${DESTDIR}${PREFIX}/lib
+	install -m 644 libhello.h ${DESTDIR}${PREFIX}/include
+
+%.o: %.c %.h
+	$(CC) -c $< -o $@ -Wall
+
+libhello.so: libhello.o
+	$(CC) -shared -o $@ $<
diff --git a/doc/examples/composition/files/libhello/libhello.c b/doc/examples/composition/files/libhello/libhello.c
new file mode 100644
index 000000000..759b33926
--- /dev/null
+++ b/doc/examples/composition/files/libhello/libhello.c
@@ -0,0 +1,9 @@
+/*
+ * libhello.c - The hello library
+ */
+#include <stdio.h>
+
+void hello(const char *person)
+{
+  printf("Hello %s\n", person);
+}
diff --git a/doc/examples/composition/files/libhello/libhello.h b/doc/examples/composition/files/libhello/libhello.h
new file mode 100644
index 000000000..f714f3659
--- /dev/null
+++ b/doc/examples/composition/files/libhello/libhello.h
@@ -0,0 +1,8 @@
+/*
+ * libhello.h - The hello library
+ */
+
+/*
+ * A function to say hello to @person
+ */
+void hello(const char *person);
diff --git a/doc/examples/composition/project.conf b/doc/examples/composition/project.conf
new file mode 100644
index 000000000..dc82df72b
--- /dev/null
+++ b/doc/examples/composition/project.conf
@@ -0,0 +1,12 @@
+# Unique project name
+name: composition
+
+# Required BuildStream format version
+format-version: 18
+
+# Subdirectory where elements are stored
+element-path: elements
+
+# Define an alias for our alpine tarball
+aliases:
+  alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/
diff --git a/doc/sessions/composition.run b/doc/sessions/composition.run
new file mode 100644
index 000000000..c19a9914e
--- /dev/null
+++ b/doc/sessions/composition.run
@@ -0,0 +1,20 @@
+
+commands:
+# Make it fetch first
+- directory: ../examples/composition
+  command: source fetch hello.bst
+
+# Capture the build output
+- directory: ../examples/composition
+  output: ../source/sessions/composition-build.html
+  command: build runtime-only.bst
+
+# List the contents
+- directory: ../examples/composition
+  output: ../source/sessions/composition-list-contents.html
+  command: artifact list-contents runtime-only.bst
+
+# Run hello world
+- directory: ../examples/composition
+  output: ../source/sessions/composition-shell.html
+  command: shell runtime-only.bst -- hello audience
diff --git a/doc/source/format_project.rst b/doc/source/format_project.rst
index 978750031..549743146 100644
--- a/doc/source/format_project.rst
+++ b/doc/source/format_project.rst
@@ -779,6 +779,8 @@ the number of jobs for a given build without affecting the resulting
 cache key.
 
 
+.. _project_split_rules:
+
 Split rules
 ~~~~~~~~~~~
 The project wide :ref:`split rules <public_split_rules>` defaults can
diff --git a/doc/source/handling-files/composition.rst b/doc/source/handling-files/composition.rst
new file mode 100644
index 000000000..729ed88ee
--- /dev/null
+++ b/doc/source/handling-files/composition.rst
@@ -0,0 +1,194 @@
+
+
+.. _handling_files_composition:
+
+Composition
+===========
+In this chapter we will explore how to create *compositions* of multiple
+input filesystem trees, using the :mod:`compose <elements.compose>` element.
+
+.. note::
+
+   This example is distributed with BuildStream
+   in the `doc/examples/composition
+   <https://gitlab.com/BuildStream/buildstream/tree/master/doc/examples/composition>`_
+   subdirectory.
+
+
+Overview
+--------
+Composing a directory tree based on a set of build dependencies is often
+one of the important steps you might perform in order to create a single artifact
+which can be checked out and deployed.
+
+In order to use the :mod:`compose <elements.compose>` element, it is important
+to first understand the concept of :ref:`split rules <public_split_rules>`, which
+we will cover in this chapter.
+
+
+Introducing split rules
+~~~~~~~~~~~~~~~~~~~~~~~
+The :ref:`split rules <public_split_rules>` of an element declaration denote
+which sets of files in the given element's resulting artifact belong to which
+*domain name*.
+
+The *domains* can then be used in various ways, using plugins which understand
+*split rule domains*.
+
+BuildStream's :ref:`default project configuration <project_builtin_defaults>`
+contains a sensible set of default *split rule domains* for the purpose of
+artifact splitting, they can be overridden in :ref:`your project.conf <project_split_rules>`,
+and finally on a per element basis in the :ref:`public data <public_builtin>`
+of your element declarations.
+
+.. note::
+
+   Projects are free to add additional *split rule domains* on top of the
+   default domains provided by the default project configuration.
+
+   There is nothing wrong with defining split rule domains which *overlap*,
+   possibly capturing some of the same files also captured by another
+   *domain*, however you should be aware of this when later using your
+   split rules with a plugin which processes them, like the
+   :mod:`compose <elements.compose>` element described in this chapter.
+
+
+Example of split rule declaration
+'''''''''''''''''''''''''''''''''
+In an element, you might need to define or extend the ``split-rules``
+in order to capture files in custom locations in a logical *domain*.
+
+Here is an example of how you might use the
+:ref:`list append directive <format_directives_list_append>`
+to append an additional rule to your ``split-rules`` list in order to
+capture additional data files which your application or library might
+want to include in the *runtime domain*:
+
+.. code:: yaml
+
+   # Add our .dat files to the runtime domain
+   public:
+     bst:
+       split-rules:
+         runtime:
+	   (>):
+           - |
+             %{datadir}/foo/*.dat
+
+Split rules are absolute paths which denote files within an artifact's root
+directory. The globbing patterns supported in split rules are defined in the
+:func:`reference documentation here <buildstream.utils.glob>`.
+
+.. important::
+
+   Note that because of variable expansion, split rules can often be
+   *resolved differently* for elements which have overridden path
+   related variables, like ``%{prefix}``.
+
+   This usually means that you do not need to explicitly extend or override
+   split rules on a specific element unless your element installs files to
+   special case locations.
+
+
+Project structure
+-----------------
+In this example we expand on the chapter about
+:ref:`integration commands <tutorial_integration_commands>`, so we will
+only discuss the files which are added or changed from that example.
+
+
+``elements/base/alpine.bst``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. literalinclude:: ../../examples/composition/elements/base/alpine.bst
+   :language: yaml
+
+Here we have modified the base runtime, so as to specify that for this
+element, we want to also include the runtime linker into the *runtime domain*.
+
+
+``elements/runtime-only.bst``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. literalinclude:: ../../examples/composition/elements/runtime-only.bst
+   :language: yaml
+
+As we can see, this :mod:`compose <elements.compose>` element has
+been configured to only include files from the *runtime domain*.
+
+
+Using the project
+-----------------
+Now that we've presented how :ref:`split rules <public_split_rules>`
+work and shown how to use them in the context of this example, lets
+use the :mod:`compose <elements.compose>` element we've created and
+observe the results.
+
+
+Building the project
+~~~~~~~~~~~~~~~~~~~~
+
+.. raw:: html
+   :file: ../sessions/composition-build.html
+
+As you can see in the output, this composition has only a few hundred
+files, but the complete ``alpine.bst`` runtime has several thousand
+files.
+
+
+List the content
+~~~~~~~~~~~~~~~~
+At the risk of this being a long list, let's :ref:`list the
+contents of this artifact <invoking_artifact_list_contents>`
+
+.. raw:: html
+   :file: ../sessions/composition-list-contents.html
+
+Some things to observe here:
+
+* The list does include the ``/usr/bin/hello`` program and
+  also the ``/usr/lib/libhello.so`` shared library.
+
+  These paths are both captured by the default split rules
+  for the *runtime domain*.
+
+* The list does not include the ``/usr/include/libhello.h``
+  header file which was used to compile ``/usr/bin/hello``.
+
+  The header file is not captured by the *runtime domain*
+  by default. It is however captured by the *devel domain*.
+
+* The runtime linker ``/lib/ld-musl-x86_64.so.1``, as this was
+  explicitly added to the *runtime domain* for the ``base/alpine.bst``
+  element which provides this file.
+
+.. tip::
+
+   The reader at this time might want to list the content of
+   other elements built from this project, such as the
+   ``hello.bst`` element by itself, or the ``base/alpine.bst``
+   element.
+
+
+Run the program
+~~~~~~~~~~~~~~~
+Finally, lets just run the program we built.
+
+.. raw:: html
+   :file: ../sessions/composition-shell.html
+
+Here we can see that we at least have the required files to run
+our hello world program, however we would not have if we were
+missing the runtime linker which we added in ``base/alpine.bst``.
+
+
+Summary
+-------
+In this chapter we've gotten familiar with :ref:`split rules <public_split_rules>`
+annotations, and we've learned enough about the :mod:`compose <elements.compose>`
+element such that we can start creating our own compositions using
+*split domains*.
+
+We've also used the :ref:`list append directive <format_directives_list_append>`
+and we are now observing the contents of artifacts using
+:ref:`bst artifact list-contents <invoking_artifact_list_contents>`.
diff --git a/doc/source/main_using.rst b/doc/source/main_using.rst
index 067e7bb16..ddda44a26 100644
--- a/doc/source/main_using.rst
+++ b/doc/source/main_using.rst
@@ -13,6 +13,7 @@ guides and information on user preferences and configuration.
 
    using_tutorial
    using_developing
+   using_handling_files
    using_junctions
    using_config
    using_commands
diff --git a/doc/source/using_handling_files.rst b/doc/source/using_handling_files.rst
new file mode 100644
index 000000000..8951764e0
--- /dev/null
+++ b/doc/source/using_handling_files.rst
@@ -0,0 +1,10 @@
+Handling files
+==============
+This section covers the various aspects related to finer grained
+handling of files.
+
+.. toctree::
+   :numbered:
+   :maxdepth: 1
+
+   handling-files/composition.rst