doc/bst2html.py, doc/Makefile: Added --force option

If --force is not specified, then we'll skip session files in
the case that all of the outputs exist.

Now setting BST_FORCE_SESSION_REBUILD when building the docs
will cause the session files to be rebuilt regardless of whether
they exist or not.

The .gitlab-ci.yml was also changed to use this and force rebuilds.

4 files changed, 66 insertions, 14 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7c5bcbd24..3c6554698 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -147,8 +147,7 @@ docs:
   - pip3 install sphinx_rtd_theme
   - cd dist && ./unpack.sh && cd buildstream
   - pip3 install .
-  - make -C doc sessions
-  - make -C doc
+  - make BST_FORCE_SESSION_REBUILD=1 -C doc
   - cd ../..
   - mv dist/buildstream/doc/build/html public
   artifacts:
diff --git a/HACKING.rst b/HACKING.rst
index 1c9624090..46245ab78 100644
--- a/HACKING.rst
+++ b/HACKING.rst
@@ -269,9 +269,9 @@ you can view in your browser locally to test.
 
 Regenerating session html
 '''''''''''''''''''''''''
-The documentation build will only build the session files if explicitly
-asked to. We revision the generated session html files in order to reduce
-the burden on documentation contributors.
+The documentation build will build the session files if they are missing,
+or if explicitly asked to rebuild. We revision the generated session html files
+in order to reduce the burden on documentation contributors.
 
 To explicitly rebuild the session snapshot html files, it is recommended that you
 first set the ``BST_SOURCE_CACHE`` environment variable to your source cache, this
@@ -279,9 +279,9 @@ will make the docs build reuse already downloaded sources::
 
   export BST_SOURCE_CACHE=~/.cache/buildstream/sources
 
-To force build the session html, simply run the following::
+To force rebuild session html while building the doc, simply build the docs like this::
 
-  make -C doc sessions
+  make BST_FORCE_SESSION_REBUILD=1 -C doc
 
 
 Man pages
diff --git a/doc/Makefile b/doc/Makefile
index 19e5ab4a3..ffccba7f1 100644
--- a/doc/Makefile
+++ b/doc/Makefile
@@ -24,7 +24,13 @@ ALLSPHINXOPTS   = -W -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS)
 # the i18n builder cannot share the environment and doctrees with the others
 I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
 
+# Set BST_FORCE_SESSION_REBUILD to force rebuild the docs
 BST2HTML = $(CURDIR)/bst2html.py
+BST2HTMLOPTS =
+ifneq ($(strip $(BST_FORCE_SESSION_REBUILD)),)
+BST2HTMLOPTS = --force
+endif
+
 
 .PHONY: all clean templates templates-clean sessions sessions-clean html devhelp
 
@@ -83,8 +89,8 @@ templates-clean:
 # if you want to regenerate them.
 #
 sessions:
-	for file in $(wildcard sessions/*.run); do	\
-	    $(BST2HTML) --description $$file;		\
+	for file in $(wildcard sessions/*.run); do		\
+	    $(BST2HTML) $(BST2HTMLOPTS) --description $$file;	\
 	done
 
 sessions-clean:
@@ -93,7 +99,7 @@ sessions-clean:
 # Targets which generate docs with sphinx build
 #
 #
-html devhelp: templates
+html devhelp: templates sessions
 	@echo "Building $@..."
 	PYTHONPATH=$(CURDIR)/../buildstream/plugins \
 	    $(SPHINXBUILD) -b $@ $(ALLSPHINXOPTS) "$(BUILDDIR)/$@" \
diff --git a/doc/bst2html.py b/doc/bst2html.py
index 9624b301f..c9e98bb36 100755
--- a/doc/bst2html.py
+++ b/doc/bst2html.py
@@ -278,11 +278,52 @@ def generate_html(output, directory, config_file, source_cache, tempdir, palette
     return final_output
 
 
-def run_session(description, tempdir, source_cache, palette, config_file):
+# check_needs_build()
+#
+# Checks whether filename, specified relative to basedir,
+# needs to be built (based on whether it exists).
+#
+# Args:
+#    basedir (str): The base directory to check relative of, or None for CWD
+#    filename (str): The basedir relative path to the file
+#    force (bool): Whether force rebuilding of existing things is enabled
+#
+# Returns:
+#    (bool): Whether the file needs to be built
+#
+def check_needs_build(basedir, filename, force=False):
+    if force:
+        return True
+
+    if basedir is None:
+        basedir = os.getcwd()
+
+    filename = os.path.join(basedir, filename)
+    filename = os.path.realpath(filename)
+    if not os.path.exists(filename):
+        return True
 
+    return False
+
+
+def run_session(description, tempdir, source_cache, palette, config_file, force):
     desc = _yaml.load(description, shortname=os.path.basename(description))
     desc_dir = os.path.dirname(description)
 
+    # Preflight commands and check if we can skip this session
+    #
+    if not force:
+        needs_build = False
+        commands = _yaml.node_get(desc, list, 'commands')
+        for command in commands:
+            output = _yaml.node_get(command, str, 'output', default_value=None)
+            if output is not None and check_needs_build(desc_dir, output, force=False):
+                needs_build = True
+                break
+        if not needs_build:
+            click.echo("Skipping '{}' as no files need to be built".format(description), err=True)
+            return
+
     # FIXME: Workaround a setuptools bug where the symlinks
     #        we store in git dont get carried into a release
     #        tarball. This workaround lets us build docs from
@@ -365,6 +406,8 @@ def run_session(description, tempdir, source_cache, palette, config_file):
 @click.option('--directory', '-C',
               type=click.Path(file_okay=False, dir_okay=True),
               help="The project directory where to run the command")
+@click.option('--force', is_flag=True, default=False,
+              help="Force rebuild, even if the file exists")
 @click.option('--source-cache',
               type=click.Path(file_okay=False, dir_okay=True),
               help="A shared source cache")
@@ -378,7 +421,7 @@ def run_session(description, tempdir, source_cache, palette, config_file):
               type=click.Path(file_okay=True, dir_okay=False, readable=True),
               help="A file describing what to do")
 @click.argument('command', type=click.STRING, nargs=-1)
-def run_bst(directory, source_cache, description, palette, output, command):
+def run_bst(directory, force, source_cache, description, palette, output, command):
     """Run a bst command and capture stdout/stderr in html
 
     This command normally takes a description yaml file, see the HACKING
@@ -387,11 +430,15 @@ def run_bst(directory, source_cache, description, palette, output, command):
     if not source_cache and os.environ.get('BST_SOURCE_CACHE'):
         source_cache = os.environ['BST_SOURCE_CACHE']
 
+    if output is not None and not check_needs_build(None, output, force=force):
+        click.echo("No need to rebuild {}".format(output))
+        return 0
+
     with workdir(source_cache=source_cache) as (tempdir, config_file, source_cache):
 
         if description:
-            run_session(description, tempdir, source_cache, palette, config_file)
-            return
+            run_session(description, tempdir, source_cache, palette, config_file, force)
+            return 0
 
         # Run a command specified on the CLI
         #