Allow source plugins to access previous sources

Source plugin implementations can now specify that they need access to
previously staged sources by specifying
`BST_REQUIRES_PREVIOUS_SOURCES_TRACK` and/or
`BST_REQUIRES_PREVIOUS_SOURCES_FETCH`, corresponding to access at `track`
and `fetch` times respectively.

Fixes #381.

Replaces !505. For relevant discussion, see this discussion:
https://gitlab.com/BuildStream/buildstream/merge_requests/505#note_83780747

5 files changed, 151 insertions, 0 deletions

diff --git a/tests/sources/previous_source_access.py b/tests/sources/previous_source_access.py
new file mode 100644
index 000000000..f7045383c
--- /dev/null
+++ b/tests/sources/previous_source_access.py
@@ -0,0 +1,47 @@
+import os
+import pytest
+
+from tests.testutils import cli
+
+DATA_DIR = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    'previous_source_access'
+)
+
+
+##################################################################
+#                              Tests                             #
+##################################################################
+# Test that plugins can access data from previous sources
+@pytest.mark.datafiles(DATA_DIR)
+def test_custom_transform_source(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+
+    # Ensure we can track
+    result = cli.run(project=project, args=[
+        'track', 'target.bst'
+    ])
+    result.assert_success()
+
+    # Ensure we can fetch
+    result = cli.run(project=project, args=[
+        'fetch', 'target.bst'
+    ])
+    result.assert_success()
+
+    # Ensure we get correct output from foo_transform
+    result = cli.run(project=project, args=[
+        'build', 'target.bst'
+    ])
+    destpath = os.path.join(cli.directory, 'checkout')
+    result = cli.run(project=project, args=[
+        'checkout', 'target.bst', destpath
+    ])
+    result.assert_success()
+    # Assert that files from both sources exist, and that they have
+    # the same content
+    assert os.path.exists(os.path.join(destpath, 'file'))
+    assert os.path.exists(os.path.join(destpath, 'filetransform'))
+    with open(os.path.join(destpath, 'file')) as file1:
+        with open(os.path.join(destpath, 'filetransform')) as file2:
+            assert file1.read() == file2.read()
diff --git a/tests/sources/previous_source_access/elements/target.bst b/tests/sources/previous_source_access/elements/target.bst
new file mode 100644
index 000000000..c9d3b9bb6
--- /dev/null
+++ b/tests/sources/previous_source_access/elements/target.bst
@@ -0,0 +1,6 @@
+kind: import
+
+sources:
+- kind: local
+  path: files/file
+- kind: foo_transform
diff --git a/tests/sources/previous_source_access/files/file b/tests/sources/previous_source_access/files/file
new file mode 100644
index 000000000..980a0d5f1
--- /dev/null
+++ b/tests/sources/previous_source_access/files/file
@@ -0,0 +1 @@
+Hello World!
diff --git a/tests/sources/previous_source_access/plugins/sources/foo_transform.py b/tests/sources/previous_source_access/plugins/sources/foo_transform.py
new file mode 100644
index 000000000..7101bfd24
--- /dev/null
+++ b/tests/sources/previous_source_access/plugins/sources/foo_transform.py
@@ -0,0 +1,87 @@
+"""
+foo_transform - transform "file" from previous sources into "filetransform"
+===========================================================================
+
+This is a test source plugin that looks for a file named "file" staged by
+previous sources, and copies its contents to a file called "filetransform".
+
+"""
+
+import os
+import hashlib
+
+from buildstream import Consistency, Source, SourceError, utils
+
+
+class FooTransformSource(Source):
+
+    # We need access to previous both at track time and fetch time
+    BST_REQUIRES_PREVIOUS_SOURCES_TRACK = True
+    BST_REQUIRES_PREVIOUS_SOURCES_FETCH = True
+
+    @property
+    def mirror(self):
+        """Directory where this source should stage its files
+
+        """
+        path = os.path.join(self.get_mirror_directory(), self.name,
+                            self.ref.strip())
+        os.makedirs(path, exist_ok=True)
+        return path
+
+    def configure(self, node):
+        self.node_validate(node, ['ref'] + Source.COMMON_CONFIG_KEYS)
+        self.ref = self.node_get_member(node, str, 'ref', None)
+
+    def preflight(self):
+        pass
+
+    def get_unique_key(self):
+        return (self.ref,)
+
+    def get_consistency(self):
+        if self.ref is None:
+            return Consistency.INCONSISTENT
+        # If we have a file called "filetransform", verify that its checksum
+        # matches our ref. Otherwise, it resolved but not cached.
+        fpath = os.path.join(self.mirror, 'filetransform')
+        try:
+            with open(fpath, 'rb') as f:
+                if hashlib.sha256(f.read()).hexdigest() == self.ref.strip():
+                    return Consistency.CACHED
+        except Exception:
+            pass
+        return Consistency.RESOLVED
+
+    def get_ref(self):
+        return self.ref
+
+    def set_ref(self, ref, node):
+        self.ref = node['ref'] = ref
+
+    def track(self, previous_sources_dir):
+        # Store the checksum of the file from previous source as our ref
+        fpath = os.path.join(previous_sources_dir, 'file')
+        with open(fpath, 'rb') as f:
+            return hashlib.sha256(f.read()).hexdigest()
+
+    def fetch(self, previous_sources_dir):
+        fpath = os.path.join(previous_sources_dir, 'file')
+        # Verify that the checksum of the file from previous source matches
+        # our ref
+        with open(fpath, 'rb') as f:
+            if hashlib.sha256(f.read()).hexdigest() != self.ref.strip():
+                raise SourceError("Element references do not match")
+
+        # Copy "file" as "filetransform"
+        newfpath = os.path.join(self.mirror, 'filetransform')
+        utils.safe_copy(fpath, newfpath)
+
+    def stage(self, directory):
+        # Simply stage the "filetransform" file
+        utils.safe_copy(os.path.join(self.mirror, 'filetransform'),
+                        os.path.join(directory, 'filetransform'))
+
+
+def setup():
+    return FooTransformSource
diff --git a/tests/sources/previous_source_access/project.conf b/tests/sources/previous_source_access/project.conf
new file mode 100644
index 000000000..1749b3dba
--- /dev/null
+++ b/tests/sources/previous_source_access/project.conf
@@ -0,0 +1,10 @@
+# Project with local source plugins
+name: foo
+
+element-path: elements
+
+plugins:
+- origin: local
+  path: plugins/sources
+  sources:
+    foo_transform: 0