Add dep_transform example

1 files changed, 192 insertions, 0 deletions

diff --git a/buildstream/plugins/sources/dep_transform.py b/buildstream/plugins/sources/dep_transform.py
new file mode 100644
index 000000000..f0d02506d
--- /dev/null
+++ b/buildstream/plugins/sources/dep_transform.py
@@ -0,0 +1,192 @@
+#
+#  Copyright Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Chandan Singh <csingh43@bloomberg.net>
+#        Gokcen Nurlu <gnurlu1@bloomberg.net>
+"""A SourceTransform implementation for staging Go dependencies via `dep`
+
+**Usage**
+
+.. code: yaml
+   # Specify the godep source kind
+   kind: dep_transform
+
+   # Optionally specify a relative staging directory
+   # directory: path/to/stage
+
+   # Specify the lock file as the ref.
+   # If no ref is provided, the new ref will be the contents of the lock file
+   # generated by running `dep ensure -no-vendor`.
+   ref: "<lock_file content>"
+..
+"""
+
+import errno
+import os
+import shutil
+from hashlib import md5
+
+from buildstream import SourceTransform, Source, SourceError
+from buildstream import utils
+from buildstream import Consistency
+
+GOPKG_LOCK_FILE = 'Gopkg.lock'
+GOPKG_VENDOR_DIR = 'vendor'
+
+
+class DepTransform(SourceTransform):
+    # pylint: disable=attribute-defined-outside-init
+
+    def configure(self, node):
+        self.node_validate(node, ['ref'] + Source.COMMON_CONFIG_KEYS)
+        self.ref = self.node_get_member(node, str, 'ref', '').strip()
+
+    @property
+    def mirror(self):
+        path = os.path.join(
+            self.get_mirror_directory(),
+            self.name,
+            md5(self.ref.encode('utf-8')).hexdigest()
+        )
+        os.makedirs(path, exist_ok=True)
+        return path
+
+    def preflight(self):
+        # Check if dep is installed, get the binaries at the same time
+        self.host_dep = utils.get_host_tool('dep')
+
+    def get_unique_key(self):
+        # TODO: This plugin's inputs are actually previous source. What should
+        # we do here?
+        return (self.ref,)
+
+    def get_consistency(self):
+        if self.ref == '':
+            return Consistency.INCONSISTENT
+        for dest in (GOPKG_LOCK_FILE, GOPKG_VENDOR_DIR):
+            dest_path = os.path.join(self.mirror, dest)
+            if not os.path.exists(dest_path):
+                return Consistency.RESOLVED
+        lock_file_path = os.path.join(self.mirror, GOPKG_LOCK_FILE)
+        with open(lock_file_path, encoding='utf-8') as lock_file:
+            if lock_file.read().strip() == self.ref:
+                return Consistency.CACHED
+        return Consistency.RESOLVED
+
+    def get_ref(self):
+        return self.ref
+
+    def set_ref(self, ref, node):
+        self.ref = node['ref'] = ref
+
+    def track(self, previous_staging_dir):
+        with self.timed_activity('Tracking DepTransform source based on previous sources'):
+            with self.tempdir() as goroot:
+                # dep refuses to work on sources that are not under
+                # GOPATH/src so we need to artificially create that directory
+                # stucture.
+                go_sources_root = os.path.join(goroot, 'src', 'project')
+                os.makedirs(os.path.dirname(go_sources_root), exist_ok=True)
+
+                shutil.move(previous_staging_dir, go_sources_root)
+                gopkg_lock_path = os.path.join(go_sources_root, GOPKG_LOCK_FILE)
+                # Check if the repo has a lock file already
+                if not os.path.isfile(gopkg_lock_path):
+                    # If doesn't, let's create a new lock file
+                    self.call([self.host_dep, 'ensure', '-no-vendor'],
+                              cwd=go_sources_root,
+                              env=dict(os.environ, GOPATH=goroot),
+                              fail='Failed to update the go dep lock file with the new references')
+
+                with open(gopkg_lock_path, encoding='utf-8') as lock_file:
+                    return lock_file.read()
+
+    def fetch(self, previous_staging_dir):
+        with self.timed_activity('Fetching DepTransform source dependencies based on Gopkg.lock'):
+            with self.tempdir() as goroot:
+                # dep refuses to work on sources that are not under
+                # GOPATH/src so we need to artificially create that directory
+                # stucture.
+                go_sources_root = os.path.join(goroot, 'src', 'project')
+                os.makedirs(os.path.dirname(go_sources_root), exist_ok=True)
+
+                shutil.move(previous_staging_dir, go_sources_root)
+
+                gopkg_lock_path = os.path.join(go_sources_root, GOPKG_LOCK_FILE)
+
+                def write_lock_file():
+                    with open(gopkg_lock_path, mode='w', encoding='utf-8') as lock_file:
+                        lock_file.write(self.get_ref())
+
+                # Is there a lock file present?
+                if os.path.isfile(gopkg_lock_path):
+                    # There is! We should ensure it is the same as the ref
+                    with open(gopkg_lock_path, encoding='utf-8') as lock_file:
+                        stripped_lock_file = lock_file.read().strip()
+                        # The current yaml parser strips this already, but let's not rely on it to do so
+                    stripped_track_ref = self.get_ref().strip()
+
+                    if stripped_lock_file != stripped_track_ref:
+                        self.warn("Tracking ref and lock file in source differ. Using tracking ref.")
+                        write_lock_file()
+                else:
+                    write_lock_file()
+
+                self.call([self.host_dep, 'ensure', '-vendor-only'],
+                          cwd=go_sources_root,
+                          env=dict(os.environ, GOPATH=goroot),
+                          fail='Failed to populate the vendor directory from the lock file')
+
+                # Copy the stuff that we really need
+                for source in (GOPKG_LOCK_FILE, GOPKG_VENDOR_DIR):
+                    source_path = os.path.join(go_sources_root, source)
+                    dest_path = os.path.join(self.mirror, source)
+                    try:
+                        os.replace(source_path, dest_path)
+                    except OSError as e:
+                        # To avoid race condition between two concurrently
+                        # running `fetch()` processes.
+                        if e.errno not in (errno.ENOTEMPTY, errno.EEXIST):
+                            raise
+                        self.warn(
+                            "{} seems already fetched. ".format(dest_path) +
+                            "Continuing with the existing source."
+                        )
+
+    def stage(self, directory):
+        with self.timed_activity("Staging DepTransform source from based on Gopkg.lock", silent_nested=True):
+            shutil.copy(os.path.join(self.mirror, GOPKG_LOCK_FILE),
+                        os.path.join(directory, GOPKG_LOCK_FILE))
+            # FIXME: Should we ever need to support one, this may
+            #        break on platforms that don't support symlinks if
+            #        a project contains broken symlinks.
+            target_vendor_dir = os.path.join(directory, GOPKG_VENDOR_DIR)
+            try:
+                shutil.copytree(os.path.join(self.mirror, GOPKG_VENDOR_DIR),
+                                target_vendor_dir,
+                                symlinks=True)
+            except FileExistsError:
+                raise SourceError("{}: Unable to stage vendor directory because it already exists here '{}'"
+                                  .format(self, target_vendor_dir))
+            except shutil.Error as err:
+                raise SourceError("{}: Unable to stage vendor directory at '{}'"
+                                  .format(self, target_vendor_dir)) from err
+
+
+# Plugin entry point
+def setup():
+    return DepTransform