Add patch source plugin

Fixes #63

23 files changed, 341 insertions, 7 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ca4385f73..0f1d3e691 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -27,7 +27,7 @@ before_script:
 pytest:
   stage: test
   script:
-  - dnf install -y bzr
+  - dnf install -y bzr patch
   - python3 setup.py test
   - mkdir -p coverage-pytest/
   - cp .coverage.* coverage-pytest/coverage.pytest
diff --git a/buildstream/plugins/sources/patch.py b/buildstream/plugins/sources/patch.py
new file mode 100644
index 000000000..895fe8cd4
--- /dev/null
+++ b/buildstream/plugins/sources/patch.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+#
+#  Copyright Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Chandan Singh <csingh43@bloomberg.net>
+
+"""A Source implementation for applying local patches
+
+**Usage:**
+
+.. code:: yaml
+
+   # Specify the local source kind
+   kind: patch
+
+   # Specify the project relative path to a patch file
+   path: files/somefile.diff
+
+   # Optionally specify the root directory for the patch
+   # directory: path/to/stage
+
+   # Optionally specify the strip level, defaults to 1
+   strip-level: 1
+
+"""
+
+import os
+import hashlib
+from buildstream import Source, SourceError, Consistency
+from buildstream import utils
+
+
+class PatchSource(Source):
+
+    def configure(self, node):
+        project = self.get_project()
+
+        self.path = self.node_get_member(node, str, "path")
+        self.strip_level = self.node_get_member(node, int, "strip-level", 1)
+        self.fullpath = os.path.join(project.directory, self.path)
+
+    def preflight(self):
+        # Check if the configured file really exists
+        if not os.path.exists(self.fullpath):
+            raise SourceError("Specified path '%s' does not exist" % self.path)
+        elif not os.path.isfile(self.fullpath):
+            raise SourceError("Specified path '%s' must be a file" % self.path)
+
+        # Check if patch is installed, get the binary at the same time
+        self.host_patch = utils.get_host_tool("patch")
+
+    def get_unique_key(self):
+        return [self.path, _sha256sum(self.fullpath), self.strip_level]
+
+    def get_consistency(self):
+        return Consistency.CACHED
+
+    def get_ref(self):
+        # We dont have a ref, we"re a local file...
+        return None
+
+    def set_ref(self, ref, node):
+        pass
+
+    def fetch(self):
+        # Nothing to do here for a local source
+        pass
+
+    def stage(self, directory):
+        with self.timed_activity("Applying local patch: {}".format(self.path)):
+            if not os.path.isdir(directory):
+                raise SourceError(
+                    "Patch directory '{}' does not exist".format(directory))
+            elif not os.listdir(directory):
+                raise SourceError("Empty patch directory '{}'".format(directory))
+            strip_level_option = "-p{}".format(self.strip_level)
+            self.call([self.host_patch, strip_level_option, "-i", self.fullpath, "-d", directory],
+                      fail="Failed to apply patch {}".format(self.path))
+
+
+# Get the sha256 sum for the content of a file
+def _sha256sum(filename):
+    h = hashlib.sha256()
+    with open(filename, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+# Plugin entry point
+def setup():
+    return PatchSource
diff --git a/tests/sources/fixture.py b/tests/sources/fixture.py
index b231d5ce7..0a32c16ba 100644
--- a/tests/sources/fixture.py
+++ b/tests/sources/fixture.py
@@ -43,12 +43,14 @@ class Setup():
         self.project._aliases['tmpdir'] = "file:///" + str(tmpdir)
         self.project._aliases['datafiles'] = "file:///" + str(datafiles)
 
-        assert(len(element.sources) == 1)
-        self.meta_source = element.sources[0]
+        assert(len(element.sources) >= 1)
 
         base = PluginBase(package='buildstream.plugins')
         self.factory = SourceFactory(base)
-        self.source = self.factory.create(self.meta_source.kind,
-                                          self.context,
-                                          self.project,
-                                          self.meta_source)
+
+        self.sources = [self.factory.create(source.kind,
+                                            self.context,
+                                            self.project,
+                                            source)
+                        for source in element.sources]
+        self.source = self.sources[0]
diff --git a/tests/sources/patch.py b/tests/sources/patch.py
new file mode 100644
index 000000000..cca2c312a
--- /dev/null
+++ b/tests/sources/patch.py
@@ -0,0 +1,136 @@
+import os
+import pytest
+
+from buildstream import SourceError
+
+# import our common fixture
+from .fixture import Setup
+
+DATA_DIR = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    'patch',
+)
+
+
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+def test_create_source(tmpdir, datafiles):
+    setup = Setup(datafiles, 'target.bst', tmpdir)
+    patch_sources = [source for source in setup.sources if source.get_kind() == 'patch']
+    assert(len(patch_sources) == 1)
+
+
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+def test_preflight(tmpdir, datafiles):
+    setup = Setup(datafiles, 'target.bst', tmpdir)
+    patch_source = [source for source in setup.sources if source.get_kind() == 'patch'][0]
+
+    # Just expect that this passes without throwing any exception
+    patch_source.preflight()
+
+
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+def test_preflight_fail(tmpdir, datafiles):
+    setup = Setup(datafiles, 'target.bst', tmpdir)
+    patch_source = [source for source in setup.sources if source.get_kind() == 'patch'][0]
+
+    # Delete the file which the local source wants
+    localfile = os.path.join(datafiles.dirname, datafiles.basename, 'file_1.patch')
+    os.remove(localfile)
+
+    # Expect a preflight error
+    with pytest.raises(SourceError):
+        patch_source.preflight()
+
+
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+def test_unique_key(tmpdir, datafiles):
+    setup = Setup(datafiles, 'target.bst', tmpdir)
+    patch_source = [source for source in setup.sources if source.get_kind() == 'patch'][0]
+
+    # Get the unique key
+    unique_key = patch_source.get_unique_key()
+
+    # No easy way to test this, let's just check that the
+    # returned 'thing' is an array of tuples and the first element
+    # of the first tuple is the filename, and the second is not falsy
+    assert(isinstance(unique_key, list))
+    filename, digest, strip_level = unique_key
+    assert(filename == 'file_1.patch')
+    assert(digest)
+    assert(strip_level == 1)
+
+
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+def test_stage_file(tmpdir, datafiles):
+    setup = Setup(datafiles, 'target.bst', tmpdir)
+
+    for source in setup.sources:
+        source.preflight()
+        source.stage(setup.context.builddir)
+    with open(os.path.join(setup.context.builddir, 'file.txt')) as f:
+        assert(f.read() == 'This is text file with superpowers\n')
+
+
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+def test_stage_file_nonexistent_dir(tmpdir, datafiles):
+    setup = Setup(datafiles, 'failure-nonexistent-dir.bst', tmpdir)
+    patch_sources = [source for source in setup.sources if source.get_kind() == 'patch']
+    assert(len(patch_sources) == 1)
+
+    for source in setup.sources:
+        source.preflight()
+        if source.get_kind() == 'patch':
+            with pytest.raises(SourceError):
+                source.stage(setup.context.builddir)
+
+
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+def test_stage_file_empty_dir(tmpdir, datafiles):
+    setup = Setup(datafiles, 'failure-empty-dir.bst', tmpdir)
+    patch_sources = [source for source in setup.sources if source.get_kind() == 'patch']
+    assert(len(patch_sources) == 1)
+
+    for source in setup.sources:
+        source.preflight()
+        if source.get_kind() == 'patch':
+            with pytest.raises(SourceError):
+                source.stage(setup.context.builddir)
+
+
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'separate-patch-dir'))
+def test_stage_separate_patch_dir(tmpdir, datafiles):
+    setup = Setup(datafiles, 'target.bst', tmpdir)
+    patch_sources = [source for source in setup.sources if source.get_kind() == 'patch']
+    assert(len(patch_sources) == 1)
+
+    for source in setup.sources:
+        source.preflight()
+        source.stage(setup.context.builddir)
+    with open(os.path.join(setup.context.builddir, 'file.txt')) as f:
+        assert(f.read() == 'This is text file in a directory with superpowers\n')
+
+
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'multiple-patches'))
+def test_stage_multiple_patches(tmpdir, datafiles):
+    setup = Setup(datafiles, 'target.bst', tmpdir)
+    patch_sources = [source for source in setup.sources if source.get_kind() == 'patch']
+    assert(len(patch_sources) == 2)
+
+    for source in setup.sources:
+        source.preflight()
+        source.stage(setup.context.builddir)
+    with open(os.path.join(setup.context.builddir, 'file.txt')) as f:
+        assert(f.read() == 'This is text file with more superpowers\n')
+
+
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'different-strip-level'))
+def test_patch_strip_level(tmpdir, datafiles):
+    setup = Setup(datafiles, 'target.bst', tmpdir)
+    patch_sources = [source for source in setup.sources if source.get_kind() == 'patch']
+    assert(len(patch_sources) == 1)
+
+    for source in setup.sources:
+        source.preflight()
+        source.stage(setup.context.builddir)
+    with open(os.path.join(setup.context.builddir, 'file.txt')) as f:
+        assert(f.read() == 'This is text file with superpowers\n')
diff --git a/tests/sources/patch/basic/failure-empty-dir.bst b/tests/sources/patch/basic/failure-empty-dir.bst
new file mode 100644
index 000000000..42fd33c19
--- /dev/null
+++ b/tests/sources/patch/basic/failure-empty-dir.bst
@@ -0,0 +1,5 @@
+kind: pony
+description: This is also the pony
+sources:
+- kind: patch
+  path: file_1.patch
diff --git a/tests/sources/patch/basic/failure-nonexistent-dir.bst b/tests/sources/patch/basic/failure-nonexistent-dir.bst
new file mode 100644
index 000000000..90270f1e3
--- /dev/null
+++ b/tests/sources/patch/basic/failure-nonexistent-dir.bst
@@ -0,0 +1,6 @@
+kind: pony
+description: This is also the pony
+sources:
+- kind: patch
+  path: file_1.patch
+  directory: /idontexist
diff --git a/tests/sources/patch/basic/file.txt b/tests/sources/patch/basic/file.txt
new file mode 100644
index 000000000..a496efee8
--- /dev/null
+++ b/tests/sources/patch/basic/file.txt
@@ -0,0 +1 @@
+This is a text file
diff --git a/tests/sources/patch/basic/file_1.patch b/tests/sources/patch/basic/file_1.patch
new file mode 100644
index 000000000..424a486dd
--- /dev/null
+++ b/tests/sources/patch/basic/file_1.patch
@@ -0,0 +1,7 @@
+diff --git a/file.txt b/file.txt
+index a496efe..341ef26 100644
+--- a/file.txt
++++ b/file.txt
+@@ -1 +1 @@
+-This is a text file
++This is text file with superpowers
diff --git a/tests/sources/patch/basic/project.conf b/tests/sources/patch/basic/project.conf
new file mode 100644
index 000000000..afa0f5475
--- /dev/null
+++ b/tests/sources/patch/basic/project.conf
@@ -0,0 +1,2 @@
+# Basic project
+name: foo
diff --git a/tests/sources/patch/basic/target.bst b/tests/sources/patch/basic/target.bst
new file mode 100644
index 000000000..ab86e389f
--- /dev/null
+++ b/tests/sources/patch/basic/target.bst
@@ -0,0 +1,7 @@
+kind: manual
+description: This is the pony
+sources:
+- kind: local
+  path: file.txt
+- kind: patch
+  path: file_1.patch
diff --git a/tests/sources/patch/different-strip-level/file.txt b/tests/sources/patch/different-strip-level/file.txt
new file mode 100644
index 000000000..a496efee8
--- /dev/null
+++ b/tests/sources/patch/different-strip-level/file.txt
@@ -0,0 +1 @@
+This is a text file
diff --git a/tests/sources/patch/different-strip-level/file_1.patch b/tests/sources/patch/different-strip-level/file_1.patch
new file mode 100644
index 000000000..ff7f7fea5
--- /dev/null
+++ b/tests/sources/patch/different-strip-level/file_1.patch
@@ -0,0 +1,7 @@
+diff --git foo/a/file.txt foo/b/file.txt
+index a496efe..341ef26 100644
+--- foo/a/file.txt
++++ foo/b/file.txt
+@@ -1 +1 @@
+-This is a text file
++This is text file with superpowers
diff --git a/tests/sources/patch/different-strip-level/project.conf b/tests/sources/patch/different-strip-level/project.conf
new file mode 100644
index 000000000..afa0f5475
--- /dev/null
+++ b/tests/sources/patch/different-strip-level/project.conf
@@ -0,0 +1,2 @@
+# Basic project
+name: foo
diff --git a/tests/sources/patch/different-strip-level/target.bst b/tests/sources/patch/different-strip-level/target.bst
new file mode 100644
index 000000000..d893c9edd
--- /dev/null
+++ b/tests/sources/patch/different-strip-level/target.bst
@@ -0,0 +1,8 @@
+kind: pony
+description: This is the pony
+sources:
+- kind: local
+  path: file.txt
+- kind: patch
+  path: file_1.patch
+  strip-level: 2
diff --git a/tests/sources/patch/multiple-patches/file.txt b/tests/sources/patch/multiple-patches/file.txt
new file mode 100644
index 000000000..a496efee8
--- /dev/null
+++ b/tests/sources/patch/multiple-patches/file.txt
@@ -0,0 +1 @@
+This is a text file
diff --git a/tests/sources/patch/multiple-patches/file_1.patch b/tests/sources/patch/multiple-patches/file_1.patch
new file mode 100644
index 000000000..424a486dd
--- /dev/null
+++ b/tests/sources/patch/multiple-patches/file_1.patch
@@ -0,0 +1,7 @@
+diff --git a/file.txt b/file.txt
+index a496efe..341ef26 100644
+--- a/file.txt
++++ b/file.txt
+@@ -1 +1 @@
+-This is a text file
++This is text file with superpowers
diff --git a/tests/sources/patch/multiple-patches/file_2.patch b/tests/sources/patch/multiple-patches/file_2.patch
new file mode 100644
index 000000000..f56614bae
--- /dev/null
+++ b/tests/sources/patch/multiple-patches/file_2.patch
@@ -0,0 +1,7 @@
+diff --git a/file.txt b/file.txt
+index a496efe..341ef26 100644
+--- a/file.txt
++++ b/file.txt
+@@ -1 +1 @@
+-This is text file with superpowers
++This is text file with more superpowers
diff --git a/tests/sources/patch/multiple-patches/project.conf b/tests/sources/patch/multiple-patches/project.conf
new file mode 100644
index 000000000..afa0f5475
--- /dev/null
+++ b/tests/sources/patch/multiple-patches/project.conf
@@ -0,0 +1,2 @@
+# Basic project
+name: foo
diff --git a/tests/sources/patch/multiple-patches/target.bst b/tests/sources/patch/multiple-patches/target.bst
new file mode 100644
index 000000000..080ee2c0d
--- /dev/null
+++ b/tests/sources/patch/multiple-patches/target.bst
@@ -0,0 +1,9 @@
+kind: pony
+description: This is the pony
+sources:
+- kind: local
+  path: file.txt
+- kind: patch
+  path: file_1.patch
+- kind: patch
+  path: file_2.patch
diff --git a/tests/sources/patch/separate-patch-dir/file_1.patch b/tests/sources/patch/separate-patch-dir/file_1.patch
new file mode 100644
index 000000000..ae8bc3381
--- /dev/null
+++ b/tests/sources/patch/separate-patch-dir/file_1.patch
@@ -0,0 +1,7 @@
+diff --git a/file.txt b/file.txt
+index a496efe..341ef26 100644
+--- a/file.txt
++++ b/file.txt
+@@ -1 +1 @@
+-This is a text file in a directory
++This is text file in a directory with superpowers
diff --git a/tests/sources/patch/separate-patch-dir/project.conf b/tests/sources/patch/separate-patch-dir/project.conf
new file mode 100644
index 000000000..afa0f5475
--- /dev/null
+++ b/tests/sources/patch/separate-patch-dir/project.conf
@@ -0,0 +1,2 @@
+# Basic project
+name: foo
diff --git a/tests/sources/patch/separate-patch-dir/target.bst b/tests/sources/patch/separate-patch-dir/target.bst
new file mode 100644
index 000000000..70ae20b70
--- /dev/null
+++ b/tests/sources/patch/separate-patch-dir/target.bst
@@ -0,0 +1,8 @@
+kind: pony
+description: This is the pony
+sources:
+- kind: local
+  path: test-dir
+- kind: patch
+  path: file_1.patch
+  directory: test-dir
diff --git a/tests/sources/patch/separate-patch-dir/test-dir/file.txt b/tests/sources/patch/separate-patch-dir/test-dir/file.txt
new file mode 100644
index 000000000..425911a7f
--- /dev/null
+++ b/tests/sources/patch/separate-patch-dir/test-dir/file.txt
@@ -0,0 +1 @@
+This is a text file in a directory