doc/examples/strict-mode: Adding a user guide for using non-strict build plans

17 files changed, 507 insertions, 0 deletions

diff --git a/doc/examples/strict-mode/elements/base.bst b/doc/examples/strict-mode/elements/base.bst
new file mode 100644
index 000000000..1b85a9e8c
--- /dev/null
+++ b/doc/examples/strict-mode/elements/base.bst
@@ -0,0 +1,5 @@
+kind: stack
+description: Base stack
+
+depends:
+- base/alpine.bst
diff --git a/doc/examples/strict-mode/elements/base/alpine.bst b/doc/examples/strict-mode/elements/base/alpine.bst
new file mode 100644
index 000000000..433f4773a
--- /dev/null
+++ b/doc/examples/strict-mode/elements/base/alpine.bst
@@ -0,0 +1,17 @@
+kind: import
+description: |
+
+    Alpine Linux base runtime
+
+sources:
+- kind: tar
+  url: alpine:integration-tests-base.v1.x86_64.tar.xz
+  ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639
+
+#
+# Run ldconfig in the libdir before running anything
+#
+public:
+  bst:
+    integration-commands:
+    - ldconfig "%{libdir}"
diff --git a/doc/examples/strict-mode/elements/hello-dynamic.bst b/doc/examples/strict-mode/elements/hello-dynamic.bst
new file mode 100644
index 000000000..8025a1ed3
--- /dev/null
+++ b/doc/examples/strict-mode/elements/hello-dynamic.bst
@@ -0,0 +1,22 @@
+kind: manual
+description: |
+
+  The dynamically linked hello application
+
+# Depend on the hello library
+depends:
+- libhello.bst
+
+# Stage the files/hello directory for building
+sources:
+  - kind: local
+    path: files/hello
+
+# Now configure the commands to run
+config:
+
+  build-commands:
+  - make -f Makefile.dynamic PREFIX="%{prefix}"
+
+  install-commands:
+  - make -f Makefile.dynamic -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install
diff --git a/doc/examples/strict-mode/elements/hello-static.bst b/doc/examples/strict-mode/elements/hello-static.bst
new file mode 100644
index 000000000..63806d184
--- /dev/null
+++ b/doc/examples/strict-mode/elements/hello-static.bst
@@ -0,0 +1,24 @@
+kind: manual
+description: |
+
+  The statically linked hello application
+
+# Depend on the hello library with the strict option
+#
+depends:
+- filename: libhello.bst
+  strict: true
+
+# Stage the files/hello directory for building
+sources:
+  - kind: local
+    path: files/hello
+
+# Now configure the commands to run
+config:
+
+  build-commands:
+  - make -f Makefile.static PREFIX="%{prefix}"
+
+  install-commands:
+  - make -f Makefile.static -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install
diff --git a/doc/examples/strict-mode/elements/libhello.bst b/doc/examples/strict-mode/elements/libhello.bst
new file mode 100644
index 000000000..53496c84c
--- /dev/null
+++ b/doc/examples/strict-mode/elements/libhello.bst
@@ -0,0 +1,22 @@
+kind: manual
+description: |
+
+  The libhello library
+
+# Depend on the base system
+depends:
+- base.bst
+
+# Stage the files/libhello directory for building
+sources:
+  - kind: local
+    path: files/libhello
+
+# Now configure the commands to run
+config:
+
+  build-commands:
+  - make PREFIX="%{prefix}"
+
+  install-commands:
+  - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install
diff --git a/doc/examples/strict-mode/files/hello/Makefile.dynamic b/doc/examples/strict-mode/files/hello/Makefile.dynamic
new file mode 100644
index 000000000..52b0f72cd
--- /dev/null
+++ b/doc/examples/strict-mode/files/hello/Makefile.dynamic
@@ -0,0 +1,12 @@
+# Sample makefile for hello.c
+#
+.PHONY: all install
+
+all: hello
+
+install: all
+	install -d ${DESTDIR}${PREFIX}/bin
+	install -m 755 hello ${DESTDIR}${PREFIX}/bin
+
+hello: hello.c
+	$(CC) -Wall -o $@ $< -lhello
diff --git a/doc/examples/strict-mode/files/hello/Makefile.static b/doc/examples/strict-mode/files/hello/Makefile.static
new file mode 100644
index 000000000..87fb51206
--- /dev/null
+++ b/doc/examples/strict-mode/files/hello/Makefile.static
@@ -0,0 +1,12 @@
+# Sample makefile for hello.c
+#
+.PHONY: all install
+
+all: hello
+
+install: all
+	install -d ${DESTDIR}${PREFIX}/bin
+	install -m 755 hello ${DESTDIR}${PREFIX}/bin
+
+hello: hello.c
+	$(CC) -Wall -o $@ $< /usr/lib/libhello.a
diff --git a/doc/examples/strict-mode/files/hello/hello.c b/doc/examples/strict-mode/files/hello/hello.c
new file mode 100644
index 000000000..83e762c29
--- /dev/null
+++ b/doc/examples/strict-mode/files/hello/hello.c
@@ -0,0 +1,20 @@
+/*
+ * hello.c - Simple hello program
+ */
+#include <stdio.h>
+#include <libhello.h>
+
+int main(int argc, char *argv[])
+{
+  const char *person = NULL;
+
+  if (argc > 1)
+    person = argv[1];
+
+  if (person)
+    hello(person);
+  else
+    hello("stranger");
+
+  return 0;
+}
diff --git a/doc/examples/strict-mode/files/libhello/Makefile b/doc/examples/strict-mode/files/libhello/Makefile
new file mode 100644
index 000000000..cdc20a30a
--- /dev/null
+++ b/doc/examples/strict-mode/files/libhello/Makefile
@@ -0,0 +1,21 @@
+# Sample makefile for hello library
+#
+.PHONY: all install
+
+all: libhello.so libhello.a
+
+install: all
+	install -d ${DESTDIR}${PREFIX}/lib
+	install -d ${DESTDIR}${PREFIX}/include
+	install -m 644 libhello.so ${DESTDIR}${PREFIX}/lib
+	install -m 644 libhello.a ${DESTDIR}${PREFIX}/lib
+	install -m 644 libhello.h ${DESTDIR}${PREFIX}/include
+
+%.o: %.c %.h
+	$(CC) -c $< -o $@ -Wall
+
+libhello.a: libhello.o
+	$(AR) rcs $@ $^
+
+libhello.so: libhello.o
+	$(CC) -shared -o $@ $<
diff --git a/doc/examples/strict-mode/files/libhello/libhello.c b/doc/examples/strict-mode/files/libhello/libhello.c
new file mode 100644
index 000000000..7d0eca340
--- /dev/null
+++ b/doc/examples/strict-mode/files/libhello/libhello.c
@@ -0,0 +1,9 @@
+/*
+ * libhello.c - The hello library
+ */
+#include <stdio.h>
+
+void hello(const char *person)
+{
+  printf("Good morning %s\n", person);
+}
diff --git a/doc/examples/strict-mode/files/libhello/libhello.h b/doc/examples/strict-mode/files/libhello/libhello.h
new file mode 100644
index 000000000..f714f3659
--- /dev/null
+++ b/doc/examples/strict-mode/files/libhello/libhello.h
@@ -0,0 +1,8 @@
+/*
+ * libhello.h - The hello library
+ */
+
+/*
+ * A function to say hello to @person
+ */
+void hello(const char *person);
diff --git a/doc/examples/strict-mode/project.conf b/doc/examples/strict-mode/project.conf
new file mode 100644
index 000000000..ac73434c8
--- /dev/null
+++ b/doc/examples/strict-mode/project.conf
@@ -0,0 +1,12 @@
+# Unique project name
+name: strict-mode
+
+# Required BuildStream format version
+format-version: 18
+
+# Subdirectory where elements are stored
+element-path: elements
+
+# Define an alias for our alpine tarball
+aliases:
+  alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/
diff --git a/doc/examples/strict-mode/update.patch b/doc/examples/strict-mode/update.patch
new file mode 100644
index 000000000..070b2422e
--- /dev/null
+++ b/doc/examples/strict-mode/update.patch
@@ -0,0 +1,9 @@
+--- libhello.c
++++ libhello.c
+@@ -5,5 +5,5 @@
+ 
+ void hello(const char *person)
+ {
+-  printf("Hello %s\n", person);
++  printf("Good morning %s\n", person);
+ }
diff --git a/doc/sessions/strict-mode.run b/doc/sessions/strict-mode.run
new file mode 100644
index 000000000..d043afb1a
--- /dev/null
+++ b/doc/sessions/strict-mode.run
@@ -0,0 +1,60 @@
+
+commands:
+
+# Build both static and dynamic versions
+- directory: ../examples/strict-mode
+  command: build hello-static.bst hello-dynamic.bst
+
+# Show all of the elements
+- directory: ../examples/strict-mode
+  output: ../source/sessions/strict-mode-show-initial.html
+  command: show hello-static.bst hello-dynamic.bst
+
+# Capture workspace open output 
+- directory: ../examples/strict-mode
+  output: ../source/sessions/strict-mode-workspace-open.html
+  command: workspace open --directory workspace_libhello libhello.bst
+
+# Apply a patch in the workspace
+- directory: ../examples/strict-mode
+  shell: True
+  command: patch workspace_libhello/libhello.c update.patch
+
+# Show hello-dynamic.bst with a modified library
+- directory: ../examples/strict-mode
+  output: ../source/sessions/strict-mode-show-dynamic-strict.html
+  command: show hello-dynamic.bst
+
+# Show hello-dynamic.bst with a modified library, without strict
+- directory: ../examples/strict-mode
+  output: ../source/sessions/strict-mode-show-dynamic-no-strict.html
+  command: --no-strict show hello-dynamic.bst
+
+# Show hello-static.bst with a modified library, without strict
+- directory: ../examples/strict-mode
+  output: ../source/sessions/strict-mode-show-static-no-strict.html
+  command: --no-strict show hello-static.bst
+
+# Build hello-dynamic.bst without strict
+- directory: ../examples/strict-mode
+  output: ../source/sessions/strict-mode-build-dynamic-no-strict.html
+  command: --no-strict build hello-dynamic.bst
+
+# Run hello-dynamic.bst without strict
+- directory: ../examples/strict-mode
+  output: ../source/sessions/strict-mode-run-dynamic-no-strict.html
+  command: --no-strict shell hello-dynamic.bst -- hello
+
+# Build hello-static.bst without strict
+- directory: ../examples/strict-mode
+  output: ../source/sessions/strict-mode-build-static-no-strict.html
+  command: --no-strict build hello-static.bst
+
+# Run hello-static.bst without strict
+- directory: ../examples/strict-mode
+  output: ../source/sessions/strict-mode-run-static-no-strict.html
+  command: --no-strict shell hello-static.bst -- hello
+
+# Discard workspace
+- directory: ../examples/strict-mode
+  command: workspace close --remove-dir libhello.bst 
diff --git a/doc/source/developing/strict-mode.rst b/doc/source/developing/strict-mode.rst
new file mode 100644
index 000000000..a3ee23183
--- /dev/null
+++ b/doc/source/developing/strict-mode.rst
@@ -0,0 +1,251 @@
+
+
+.. _developing_strict_mode:
+
+Strict mode
+===========
+In this section, we will cover the usage of :ref:`strict vs non-strict <user_config_strict_mode>`
+build plans in conjunction with :ref:`workspaces <developing_workspaces>`, and how this
+can help to improve your edit/compile/test cycles.
+
+.. note::
+
+   This example is distributed with BuildStream
+   in the `doc/examples/strict-mode
+   <https://gitlab.com/BuildStream/buildstream/tree/master/doc/examples/strict-mode>`_
+   subdirectory.
+
+
+Overview
+--------
+When working with BuildStream to create integrations, it is typical that you have a
+lot of components to build, and you frequently need to modify a component
+at various levels of the stack. When developing one or more applications, you might
+want to open a workspace and fix a bug in an application, or you might need to
+open a workspace on a low level shared library to fix the behavior of one or
+more misbehaving applications.
+
+By default, BuildStream will always choose to be deterministic in order to
+produce the most correct build results as possible. As such, modifying a low
+level library will result in rebuilding all of it's reverse dependencies, but
+this can be very time consuming and inconvenient for your edit/compile/test
+cycles.
+
+This is when enabling :ref:`non-strict build plans <user_config_strict_mode>`
+can be helpful.
+
+To illustrate the facets of how this works, this example will present a project
+consisting of an application which is linked both statically and dynamically
+linked to a common library.
+
+
+Project structure
+-----------------
+This project is mostly based on the :ref:`integration commands <tutorial_integration_commands>`
+example, as such we will ignore large parts of this project and only focus
+on the elements which are of specific interest.
+
+To illustrate the relationship of these two applications and the library,
+let's briefly take a look at the underlying Makefiles which are used in this
+project, starting with the library and followed by both Makefiles used to
+build the application.
+
+
+``files/libhello/Makefile``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. literalinclude:: ../../examples/strict-mode/files/libhello/Makefile
+   :language: Makefile
+
+
+``files/hello/Makefile.dynamic``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. literalinclude:: ../../examples/strict-mode/files/hello/Makefile.dynamic
+   :language: Makefile
+
+
+``files/hello/Makefile.static``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. literalinclude:: ../../examples/strict-mode/files/hello/Makefile.static
+   :language: Makefile
+
+As we can see, we have a library that is distributed both as the dynamic
+library ``libhello.so`` and also as the static archive ``libhello.a``.
+
+Now let's take a look at the two separate elements which build the
+application, first the dynamically linked version and then the static one.
+
+
+``elements/hello-dynamic.bst``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. literalinclude:: ../../examples/strict-mode/elements/hello-dynamic.bst
+   :language: yaml
+
+Nothing very special to observe about this hello program, just a
+:mod:`manual <elements.manual>` element quite similar to the one we've
+already seen in the :ref:`running commands <tutorial_running_commands>`
+example.
+
+
+``elements/hello-static.bst``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. literalinclude:: ../../examples/strict-mode/elements/hello-static.bst
+   :language: yaml
+
+Almost the same as the dynamic element, except here we have declared the
+dependency to the ``libhello.bst`` element differently: this time we have enabled
+the ``strict`` option in the :ref:`dependency declaration <format_dependencies>`.
+
+The side effect of setting this option is that ``hello-static.bst`` will be
+rebuilt any time that ``libhello.bst`` has changed, even when
+:ref:`non-strict build plans <user_config_strict_mode>` have been enabled.
+
+.. tip::
+
+   Some element plugins are designed to consume the content of their
+   dependencies entirely, and output an artifact without any transient
+   runtime dependencies, an example of this is the :mod:`compose <elements.compose>`
+   element.
+
+   In cases such as :mod:`compose <elements.compose>`, it is not necessary to
+   explicitly annotate their dependencies as ``strict``.
+
+   It is only helpful to set the ``strict`` attribute on a
+   :ref:`dependency declaration <format_dependencies>` in the case that the
+   specific dependency relationship causes data to be consumed verbatim,
+   as is the case with static linking.
+
+
+Using the project
+-----------------
+For the sake of brevity, let's assume that you've already built all of the
+elements of this project, and that you want to make some changes to the
+``libhello.bst`` element, and test how it might effect the hello program.
+
+
+Everything is already built
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. raw:: html
+   :file: ../sessions/strict-mode-show-initial.html
+
+
+Open a workspace and modify libhello.c
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Now let's open up a workspace on the hello library
+
+.. raw:: html
+   :file: ../sessions/strict-mode-workspace-open.html
+
+And go ahead and make a modification like this:
+
+.. literalinclude:: ../../examples/strict-mode/update.patch
+    :language: diff
+
+
+Observing ``hello-dynamic.bst``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Let's take a look at the :ref:`bst show <invoking_show>` output for
+the dynamically linked ``hello-dynamic.bst`` element.
+
+.. raw:: html
+   :file: ../sessions/strict-mode-show-dynamic-strict.html
+
+As one might expect, the ``libhello.bst`` element is ready to be built
+after having been modified, and the ``hello-dynamic.bst`` element is
+waiting for ``libhello.bst`` to be built before it can build.
+
+Now let's take a look at the same elements if we pass the ``--no-strict``
+option to ``bst``:
+
+.. raw:: html
+   :file: ../sessions/strict-mode-show-dynamic-no-strict.html
+
+Note that this time, the ``libhello.bst`` still needs to be built,
+but the ``hello-dymamic.bst`` element is showing up as ``cached``.
+
+.. tip::
+
+   The :ref:`bst show <invoking_show>` output will show some cache
+   keys dimmed out in the case that they are not entirely deterministic.
+
+   Here we can see that ``hello-dynamic.bst`` is dimmed out because
+   it will not be rebuilt against the changed ``libhello.bst`` element,
+   and it also has a different cache key because of this.
+
+
+Observing ``hello-static.bst``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Now let's observe the ``hello-static.bst`` element with strict mode
+disabled:
+
+.. raw:: html
+   :file: ../sessions/strict-mode-show-static-no-strict.html
+
+Note that in this case the ``hello-strict.bst`` is going to be
+rebuilt even in strict mode. This is because we annotated the
+declaration of the ``libhello.bst`` dependency with the ``strict``
+attribute.
+
+We did this because ``hello-strict.bst`` consumes the input of
+``libhello.bst`` verbatim, by way of statically linking to it, instead
+of merely being affected by the content of ``libhello.bst`` at runtime,
+as would be the case of static linking.
+
+
+Building and running ``hello-dynamic.bst``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Now let's build ``hello-dynamic.bst`` with strict mode disabled.
+
+.. raw:: html
+   :file: ../sessions/strict-mode-build-dynamic-no-strict.html
+
+Note that the :ref:`bst build <invoking_build>` command completed without
+having to build ``hello-dynamic.bst`` at all.
+
+And now we can also run ``hello-dynamic.bst``
+
+.. raw:: html
+   :file: ../sessions/strict-mode-run-dynamic-no-strict.html
+
+When running ``hello-dynamic.bst`` with no-strict mode, we are
+actually reusing the old build of ``hello-dynamic.bst`` staged against
+the new build of the modified ``libhello.bst`` element.
+
+
+Building and running ``hello-static.bst``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Finally, if we build ``hello-static.bst`` with strict mode disabled,
+we can see that it will be rebuilt regardless of strict mode being
+enabled.
+
+.. raw:: html
+   :file: ../sessions/strict-mode-build-static-no-strict.html
+
+This is of course because we declared its dependency on ``libhello.bst``
+as a ``strict`` dependency.
+
+And by the same virtue, we can see that when we run the example
+it has properly relinked against the changed static archive, and
+has the updated text in the greeting:
+
+.. raw:: html
+   :file: ../sessions/strict-mode-run-static-no-strict.html
+
+
+Summary
+-------
+In this chapter we've explored how to use :ref:`non-strict build plans <user_config_strict_mode>`
+in order to avoid rebuilding reverse dependencies of a lower level
+element you might be working with in a :ref:`workspace <invoking_workspace_open>`,
+consequently improving your edit/compile/test experience.
+
+We've also explained how to ensure your project still works properly
+with non-strict build plans when some elements perform static linking
+(or other operations which consume data from their dependencies
+verbatim), by annotating :ref:`dependency declarations <format_dependencies>`
+as ``strict``.
diff --git a/doc/source/tutorial/integration-commands.rst b/doc/source/tutorial/integration-commands.rst
index ead5be8b2..558d3d57f 100644
--- a/doc/source/tutorial/integration-commands.rst
+++ b/doc/source/tutorial/integration-commands.rst
@@ -1,5 +1,7 @@
 
 
+.. _tutorial_integration_commands:
+
 Integration commands
 ====================
 Sometimes a software requires more configuration or processing than what is
diff --git a/doc/source/using_developing.rst b/doc/source/using_developing.rst
index 4bf1ebe83..b80cb20b7 100644
--- a/doc/source/using_developing.rst
+++ b/doc/source/using_developing.rst
@@ -9,3 +9,4 @@ modules in the context of an integrated product.
    :maxdepth: 1
 
    developing/workspaces.rst
+   developing/strict-mode.rst