Add support for doing incremental builds

This functionality is only supported for sources which have an open
workspace. When such sources are present, the workspace directory will
be mounted directly inside the sandbox. As opposed to the default
behavior, which is to copy files inside the sandbox.

This will save time when building large projects as only those files
will need be re-compiled that have been modified during two consecutive
builds (assuming the underlying build system supports such behavior).

A few things to note regarding this behavior:

- If there are any `configure-commands` present, they will run only once
  for each open workspace. If an element has multiple workspaces and any
  one of them is opened/closed, they will be executed again on the next
  run. But, modifying the contents of a workspace will not trigger the
  `configure-commands` to be executed on the next run.

- Workspaced builds still leverage the cache. So, if no changes are made
  to the workspace, i.e. no files are modified, then it will not force a
  rebuild.

Fixes #192.

15 files changed, 275 insertions, 18 deletions

diff --git a/buildstream/_scheduler/buildqueue.py b/buildstream/_scheduler/buildqueue.py
index 74a7544c0..ffa99a92d 100644
--- a/buildstream/_scheduler/buildqueue.py
+++ b/buildstream/_scheduler/buildqueue.py
@@ -30,6 +30,10 @@ class BuildQueue(Queue):
     complete_name = "Built"
     queue_type = QueueType.BUILD
 
+    def prepare(self, element):
+        # Inform element in main process that it is scheduled for assembly
+        element._schedule_assemble()
+
     def process(self, element):
         element._assemble()
         return element._get_unique_id()
@@ -47,8 +51,9 @@ class BuildQueue(Queue):
         return QueueStatus.READY
 
     def done(self, element, result, returncode):
-        # Elements are cached after they are successfully assembled
-        if returncode == 0:
-            element._update_state()
+        # Inform element in main process that assembly is done
+        element._assemble_done()
+
+        element._update_state()
 
         return True
diff --git a/buildstream/_scheduler/queue.py b/buildstream/_scheduler/queue.py
index 46640677f..1b4990100 100644
--- a/buildstream/_scheduler/queue.py
+++ b/buildstream/_scheduler/queue.py
@@ -117,6 +117,16 @@ class Queue():
     def status(self, element):
         return QueueStatus.READY
 
+    # prepare()
+    #
+    # Abstract method for handling job preparation in the main process.
+    #
+    # Args:
+    #    element (Element): The element which is scheduled
+    #
+    def prepare(self, element):
+        pass
+
     # done()
     #
     # Abstract method for handling a successful job completion.
@@ -178,6 +188,8 @@ class Queue():
                 self.skipped_elements.append(element)
                 continue
 
+            self.prepare(element)
+
             job = Job(scheduler, element, self.action_name)
             scheduler.job_starting(job)
 
diff --git a/buildstream/element.py b/buildstream/element.py
index 480caea88..463c6c097 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -516,10 +516,7 @@ class Element(Plugin):
            directory (str): An absolute path within the sandbox to stage the sources at
         """
 
-        sandbox_root = sandbox.get_directory()
-        host_directory = os.path.join(sandbox_root, directory.lstrip(os.sep))
-
-        self._stage_sources_at(host_directory)
+        self._stage_sources_in_sandbox(sandbox, directory)
 
     def get_public_data(self, domain):
         """Fetch public data on this element
@@ -749,6 +746,28 @@ class Element(Plugin):
 
         self._update_state()
 
+    # _schedule_assemble():
+    #
+    # This is called in the main process before the element is assembled
+    # in a subprocess.
+    #
+    def _schedule_assemble(self):
+        for source in self.__sources:
+            source._schedule_assemble()
+
+        self._update_state()
+
+    # _assemble_done():
+    #
+    # This is called in the main process after the element has been assembled
+    # in a subprocess.
+    #
+    def _assemble_done(self):
+        for source in self.__sources:
+            source._assemble_done()
+
+        self._update_state()
+
     # _cached():
     #
     # Returns:
@@ -1051,7 +1070,7 @@ class Element(Plugin):
                 _yaml.dump(_yaml.node_sanitize(self.__dynamic_public), os.path.join(metadir, 'public.yaml'))
 
                 # ensure we have cache keys
-                self._update_state()
+                self._assemble_done()
 
                 # Store artifact metadata
                 dependencies = {
@@ -1331,20 +1350,53 @@ class Element(Plugin):
             # Run shells with network enabled and readonly root.
             return sandbox.run(argv, flags, env=environment)
 
+    # _stage_sources_in_sandbox():
+    #
+    # Stage this element's sources to a directory inside sandbox
+    #
+    # Args:
+    #     sandbox (:class:`.Sandbox`): The build sandbox
+    #     directory (str): An absolute path to stage the sources at
+    #     mount_workspaces (bool): mount workspaces if True, copy otherwise
+    #
+    def _stage_sources_in_sandbox(self, sandbox, directory, mount_workspaces=True):
+
+        if mount_workspaces:
+            # First, mount sources that have an open workspace
+            sources_to_mount = [source for source in self.sources() if source._has_workspace()]
+            for source in sources_to_mount:
+                mount_point = source._get_staging_path(directory)
+                mount_source = source._get_workspace_path()
+                sandbox.mark_directory(mount_point)
+                sandbox._set_mount_source(mount_point, mount_source)
+
+        # Stage all sources that need to be copied
+        sandbox_root = sandbox.get_directory()
+        host_directory = os.path.join(sandbox_root, directory.lstrip(os.sep))
+        self._stage_sources_at(host_directory, mount_workspaces=mount_workspaces)
+
     # _stage_sources_at():
     #
     # Stage this element's sources to a directory
     #
     # Args:
     #     directory (str): An absolute path to stage the sources at
+    #     mount_workspaces (bool): mount workspaces if True, copy otherwise
     #
-    def _stage_sources_at(self, directory):
+    def _stage_sources_at(self, directory, mount_workspaces=True):
         with self.timed_activity("Staging sources", silent_nested=True):
 
             if os.path.isdir(directory) and os.listdir(directory):
                 raise ElementError("Staging directory '{}' is not empty".format(directory))
 
-            for source in self.__sources:
+            # If mount_workspaces is set, sources with workspace are mounted
+            # directly inside the sandbox so no need to stage them here.
+            if mount_workspaces:
+                sources = [source for source in self.sources() if not source._has_workspace()]
+            else:
+                sources = self.sources()
+
+            for source in sources:
                 source._stage(directory)
 
         # Ensure deterministic mtime of sources at build time
@@ -1411,6 +1463,18 @@ class Element(Plugin):
             # Tracking is still pending
             return
 
+        if any([not source._stable() for source in self.__sources]):
+            # If any source is not stable, discard current cache key values
+            # as their correct values can only be calculated once the build is complete
+            self.__cache_key_dict = None
+            self.__cache_key = None
+            self.__weak_cache_key = None
+            self.__strict_cache_key = None
+            self.__strong_cached = None
+            self.__remotely_cached = None
+            self.__remotely_strong_cached = None
+            return
+
         if self.__weak_cache_key is None:
             # Calculate weak cache key
             # Weak cache key includes names of direct build dependencies
diff --git a/buildstream/plugins/elements/import.py b/buildstream/plugins/elements/import.py
index 3ab7f7833..23cad9cb9 100644
--- a/buildstream/plugins/elements/import.py
+++ b/buildstream/plugins/elements/import.py
@@ -64,7 +64,8 @@ class ImportElement(BuildElement):
     def assemble(self, sandbox):
 
         # Stage sources into the input directory
-        self.stage_sources(sandbox, 'input')
+        # Do not mount workspaces as the files are copied from outside the sandbox
+        self._stage_sources_in_sandbox(sandbox, 'input', mount_workspaces=False)
 
         rootdir = sandbox.get_directory()
         inputdir = os.path.join(rootdir, 'input')
diff --git a/buildstream/sandbox/_sandboxbwrap.py b/buildstream/sandbox/_sandboxbwrap.py
index f926e3de2..71fd6951b 100644
--- a/buildstream/sandbox/_sandboxbwrap.py
+++ b/buildstream/sandbox/_sandboxbwrap.py
@@ -116,9 +116,13 @@ class SandboxBwrap(Sandbox):
 
         # Add bind mounts to any marked directories
         marked_directories = self._get_marked_directories()
+        mount_source_overrides = self._get_mount_sources()
         for mark in marked_directories:
             mount_point = mark['directory']
-            mount_source = mount_map.get_mount_source(mount_point)
+            if mount_point in mount_source_overrides:
+                mount_source = mount_source_overrides[mount_point]
+            else:
+                mount_source = mount_map.get_mount_source(mount_point)
             bwrap_command += ['--bind', mount_source, mount_point]
 
         if flags & SandboxFlags.ROOT_READ_ONLY:
diff --git a/buildstream/sandbox/_sandboxchroot.py b/buildstream/sandbox/_sandboxchroot.py
index 0a6043692..584f0e116 100644
--- a/buildstream/sandbox/_sandboxchroot.py
+++ b/buildstream/sandbox/_sandboxchroot.py
@@ -246,7 +246,11 @@ class SandboxChroot(Sandbox):
 
         @contextmanager
         def mount_point(point, **kwargs):
-            mount_source = self.mount_map.get_mount_source(point)
+            mount_source_overrides = self._get_mount_sources()
+            if point in mount_source_overrides:
+                mount_source = mount_source_overrides[point]
+            else:
+                mount_source = self.mount_map.get_mount_source(point)
             mount_point = os.path.join(rootfs, point.lstrip(os.sep))
 
             with Mounter.bind_mount(mount_point, src=mount_source, stdout=stdout, stderr=stderr, **kwargs):
diff --git a/buildstream/sandbox/sandbox.py b/buildstream/sandbox/sandbox.py
index 64352d623..00245309a 100644
--- a/buildstream/sandbox/sandbox.py
+++ b/buildstream/sandbox/sandbox.py
@@ -82,6 +82,7 @@ class Sandbox():
         self.__directories = []
         self.__cwd = None
         self.__env = None
+        self.__mount_sources = {}
 
         # Setup the directories
         self.__directory = directory
@@ -195,6 +196,25 @@ class Sandbox():
     def _get_marked_directories(self):
         return self.__directories
 
+    # _get_mount_source()
+    #
+    # Fetches the list of mount sources
+    #
+    # Returns:
+    #    (dict): A dictionary where keys are mount points and values are the mount sources
+    def _get_mount_sources(self):
+        return self.__mount_sources
+
+    # _set_mount_source()
+    #
+    # Sets the mount source for a given mountpoint
+    #
+    # Args:
+    #    mountpoint (str): The absolute mountpoint path inside the sandbox
+    #    mount_source (str): the host path to be mounted at the mount point
+    def _set_mount_source(self, mountpoint, mount_source):
+        self.__mount_sources[mountpoint] = mount_source
+
     # _get_environment()
     #
     # Fetches the environment variables for running commands
diff --git a/buildstream/source.py b/buildstream/source.py
index 4882415f4..20d54ed47 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -84,6 +84,7 @@ class Source(Plugin):
         self.__origin_filename = meta.origin_filename   # Filename of the file the source was loaded from
         self.__consistency = Consistency.INCONSISTENT   # Cached consistency state
         self.__tracking = False                         # Source is scheduled to be tracked
+        self.__assemble_scheduled = False               # Source is scheduled to be assembled
         self.__workspace = None                         # Directory of the currently active workspace
         self.__workspace_key = None                     # Cached directory content hashes for workspaced source
 
@@ -301,16 +302,50 @@ class Source(Plugin):
     def _schedule_tracking(self):
         self.__tracking = True
 
+    # _schedule_assemble():
+    #
+    # This is called in the main process before the element is assembled
+    # in a subprocess.
+    #
+    def _schedule_assemble(self):
+        assert(not self.__assemble_scheduled)
+        self.__assemble_scheduled = True
+
+        # Invalidate workspace key as the build modifies the workspace directory
+        self.__workspace_key = None
+
+    # _assemble_done():
+    #
+    # This is called in the main process after the element has been assembled
+    # in a subprocess.
+    #
+    def _assemble_done(self):
+        assert(self.__assemble_scheduled)
+        self.__assemble_scheduled = False
+
+    # _stable():
+    #
+    # Unstable sources are mounted read/write and thus cannot produce a
+    # (stable) cache key before the build is complete.
+    #
+    def _stable(self):
+        # Source directory is modified by workspace build process
+        return not (self._has_workspace() and self.__assemble_scheduled)
+
     # Wrapper function around plugin provided fetch method
     #
     def _fetch(self):
         self.fetch()
 
-    # Ensures a fully constructed path and returns it
-    def _ensure_directory(self, directory):
+    # Return the path where this source should be staged under given dierctory
+    def _get_staging_path(self, directory):
         if self.__directory is not None:
             directory = os.path.join(directory, self.__directory.lstrip(os.sep))
+        return directory
 
+    # Ensures a fully constructed path and returns it
+    def _ensure_directory(self, directory):
+        directory = self._get_staging_path(directory)
         try:
             os.makedirs(directory, exist_ok=True)
         except OSError as e:
@@ -324,12 +359,12 @@ class Source(Plugin):
     # 'directory' option
     #
     def _stage(self, directory):
-        directory = self._ensure_directory(directory)
+        staging_directory = self._ensure_directory(directory)
 
         if self._has_workspace():
-            self._stage_workspace(directory)
+            self._stage_workspace(staging_directory)
         else:
-            self.stage(directory)
+            self.stage(staging_directory)
 
     # Wrapper for init_workspace()
     def _init_workspace(self, directory):
@@ -429,6 +464,8 @@ class Source(Plugin):
     # new calculation to happen by setting the 'recalculate' flag.
     #
     def _get_workspace_key(self, recalculate=False):
+        assert(not self.__assemble_scheduled)
+
         if recalculate or self.__workspace_key is None:
             fullpath = self._get_workspace_path()
 
diff --git a/integration-tests/workspace-mount-test/elements/dependencies/base-platform.bst b/integration-tests/workspace-mount-test/elements/dependencies/base-platform.bst
new file mode 100644
index 000000000..a04a5b81c
--- /dev/null
+++ b/integration-tests/workspace-mount-test/elements/dependencies/base-platform.bst
@@ -0,0 +1,19 @@
+kind: import
+description: Import the base freedesktop platform
+sources:
+- kind: ostree
+  url: gnomesdk:repo/
+  gpg-key: keys/gnome-sdk.gpg
+  (?):
+  - arch == "x86_64":
+      track: runtime/org.freedesktop.BasePlatform/x86_64/1.4
+      ref: c9d09b7250a12ef09d95952fc4f49a35e5f8c2c1dd7141b7eeada4069e6f6576
+  - arch == "i386":
+      track: runtime/org.freedesktop.BasePlatform/i386/1.4
+      ref: 27ebae91839a454596a273391b0e53063eaa8aca4fc9cb64654582bfbc338c96
+config:
+  source: files
+public:
+  bst:
+    integration-commands:
+    - ldconfig
diff --git a/integration-tests/workspace-mount-test/elements/dependencies/base-sdk.bst b/integration-tests/workspace-mount-test/elements/dependencies/base-sdk.bst
new file mode 100644
index 000000000..a1b6c5856
--- /dev/null
+++ b/integration-tests/workspace-mount-test/elements/dependencies/base-sdk.bst
@@ -0,0 +1,16 @@
+kind: import
+description: Import the base freedesktop SDK
+sources:
+- kind: ostree
+  url: gnomesdk:repo/
+  gpg-key: keys/gnome-sdk.gpg
+  (?):
+  - arch == "x86_64":
+      track: runtime/org.freedesktop.BaseSdk/x86_64/1.4
+      ref: 0d9d255d56b08aeaaffb1c820eef85266eb730cb5667e50681185ccf5cd7c882
+  - arch == "i386":
+      track: runtime/org.freedesktop.BaseSdk/i386/1.4
+      ref: 16036b747c1ec8e7fe291f5b1f667cb942f0267d08fcad962e9b7627d6cf1981
+config:
+  source: files
+  target: usr
diff --git a/integration-tests/workspace-mount-test/elements/workspace-mount-test.bst b/integration-tests/workspace-mount-test/elements/workspace-mount-test.bst
new file mode 100644
index 000000000..31d99d5f3
--- /dev/null
+++ b/integration-tests/workspace-mount-test/elements/workspace-mount-test.bst
@@ -0,0 +1,16 @@
+kind: manual
+description: workspace mount test
+
+depends:
+- filename: dependencies/base-platform.bst
+  type: build
+- filename: dependencies/base-sdk.bst
+  type: build
+
+sources:
+- kind: local
+  path: src
+
+config:
+  build-commands:
+  - g++ -c hello.cpp
diff --git a/integration-tests/workspace-mount-test/keys/gnome-sdk.gpg b/integration-tests/workspace-mount-test/keys/gnome-sdk.gpg
new file mode 100644
index 000000000..8434b686c
--- /dev/null
+++ b/integration-tests/workspace-mount-test/keys/gnome-sdk.gpg
diff --git a/integration-tests/workspace-mount-test/project.conf b/integration-tests/workspace-mount-test/project.conf
new file mode 100644
index 000000000..46985b075
--- /dev/null
+++ b/integration-tests/workspace-mount-test/project.conf
@@ -0,0 +1,21 @@
+# Import-test BuildStream project configuration.
+
+# Project name
+#
+name: script-test
+
+aliases:
+  gnomesdk: https://sdk.gnome.org/
+
+# Base project relative element path, elements will be loaded
+# from this base.
+
+element-path: elements
+
+options:
+  arch:
+    type: arch
+    description: The machine architecture
+    values:
+    - x86_64
+    - i386
diff --git a/integration-tests/workspace-mount-test/run-workspace-mount-test.sh b/integration-tests/workspace-mount-test/run-workspace-mount-test.sh
new file mode 100644
index 000000000..efa425acb
--- /dev/null
+++ b/integration-tests/workspace-mount-test/run-workspace-mount-test.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+#
+# A script to run a BuildStream test case.
+
+
+TEST_DIR="elements/"
+
+set -eu
+
+# run_test
+#
+# Run tests for this test case.
+#
+run_test () {
+    local element
+    local workspace_dir
+
+    source ../lib.sh
+
+    mkdir -p "$TEST_DIR"
+    element=workspace-mount-test.bst
+    workspace_dir="$TEST_DIR"workspace-dir
+
+    bst_with_flags workspace open "$element" "$workspace_dir"
+    bst_with_flags build "$element"
+    if [ ! -f "$workspace_dir/hello.o" ]; then
+        return 1
+    fi
+}
+
+run_test "$@"
diff --git a/integration-tests/workspace-mount-test/src/hello.cpp b/integration-tests/workspace-mount-test/src/hello.cpp
new file mode 100644
index 000000000..5d364c3cb
--- /dev/null
+++ b/integration-tests/workspace-mount-test/src/hello.cpp
@@ -0,0 +1,7 @@
+#include <iostream>
+
+int main() {
+    std::cout << "Hello world!\n";
+
+    return 0;
+}