Merge branch 'benbrown/lfs' into 'master'

Add support for Git LFS repositories

See merge request !318

5 files changed, 73 insertions, 1 deletions

diff --git a/README.md b/README.md
index ddfb1d6..6d8c446 100644
--- a/README.md
+++ b/README.md
@@ -180,6 +180,13 @@ Config for kbas follows the same approach as ybd, defaulting to config in
 NOTE: the default password is 'insecure' and the uploading is disabled unless
 you change it.
 
+### Git LFS Support
+
+ybd supports repositories that manage their binaries via
+[Git LFS](https://github.com/git-lfs/git-lfs).  Installation of the git-lfs
+binary is required in order to make use of this feature, otherwise ybd will exit
+with an error.
+
 ### Concourse Pipelines
 
 [WORK IN PROGRESS] ybd can generate concourse pipelines - see the code at
diff --git a/ybd/app.py b/ybd/app.py
index c03cda3..d748316 100644
--- a/ybd/app.py
+++ b/ybd/app.py
@@ -124,6 +124,9 @@ def warning_handler(message, category, filename, lineno, file=None, line=None):
 
 def setup(program, target, arch, mode, original_cwd=""):
     os.environ['LANG'] = 'en_US.UTF-8'
+    # Installation of git-lfs on a system can pollute /etc/gitconfig. Setting
+    # GIT_CONFIG_NOSYSTEM so ybd will ignore the system config.
+    os.environ['GIT_CONFIG_NOSYSTEM'] = "1"
     config['start-time'] = datetime.datetime.now()
     config['program'] = os.path.basename(program)
     config['my-version'] = get_version(os.path.dirname(__file__))
diff --git a/ybd/config/ybd.conf b/ybd/config/ybd.conf
index 569e44a..8b984b2 100644
--- a/ybd/config/ybd.conf
+++ b/ybd/config/ybd.conf
@@ -61,7 +61,8 @@ aliases:
 #    bug where ybd had been skipping errors in the middle of code blocks,
 # 7: (after e7be39bf) see https://gitlab.com/baserock/ybd/issues/249
 #    we need to include max-jobs in the cache-key
-artifact-version: 7
+# 8: (after c59d65cf) support added for git-lfs
+artifact-version: 8
 
 # path to be used in default chroots for builds
 base-path: ['/usr/bin', '/bin', '/usr/sbin', '/sbin']
diff --git a/ybd/repos.py b/ybd/repos.py
index cbf53a0..2b32713 100644
--- a/ybd/repos.py
+++ b/ybd/repos.py
@@ -17,6 +17,7 @@
 import contextlib
 import os
 import re
+import errno
 import shutil
 import string
 from subprocess import call, check_output
@@ -195,6 +196,26 @@ def checkout(dn):
     utils.set_mtime_recursively(dn['checkout'])
 
 
+def fetch_lfs_binaries(name, checkout):
+    app.log(name, "Checking out lfs binaries")
+    try:
+        with open(os.devnull, 'w') as fnull:
+            call(['git-lfs', 'version'], stdout=fnull, stderr=fnull)
+    except OSError as e:
+        if e.errno == errno.ENOENT:
+            app.log(name,
+                    'git-lfs is required to fetch LFS repos', exit=True)
+
+    with app.chdir(checkout), open(os.devnull, 'w') as fnull:
+        if call(['git', 'lfs', 'install', '--local'],
+                stdout=fnull, stderr=fnull):
+            app.log(name, 'lfs install failed for', checkout, exit=True)
+        if call(['git', 'lfs', 'fetch'], stdout=fnull, stderr=fnull):
+            app.log(name, 'lfs fetch failed for', checkout, exit=True)
+        if call(['git', 'lfs', 'checkout'], stdout=fnull, stderr=fnull):
+            app.log(name, 'lfs checkout failed for', checkout, exit=True)
+
+
 def _checkout(name, repo, ref, checkout):
     gitdir = os.path.join(app.config['gits'], get_repo_name(repo))
     if not os.path.exists(gitdir):
@@ -203,6 +224,10 @@ def _checkout(name, repo, ref, checkout):
         update_mirror(name, repo, gitdir)
     # checkout the required version from git
     with open(os.devnull, "w") as fnull:
+        # Ensure lfs filters are not set in .gitconfig: cloning with lfs
+        # from a local path will error.
+        call(['git', 'config', '--global', '--remove-section', 'filter.lfs'],
+             stdout=fnull, stderr=fnull)
         # We need to pass '--no-hardlinks' because right now there's nothing to
         # stop the build from overwriting the files in the .git directory
         # inside the sandbox. If they were hardlinks, it'd be possible for a
@@ -219,6 +244,10 @@ def _checkout(name, repo, ref, checkout):
 
             app.log(name, 'Git checkout %s in %s' % (repo, checkout))
             app.log(name, 'Upstream version %s' % get_version(checkout, ref))
+            is_lfs = utils.ref_expects_lfs(gitdir, ref)
+            if is_lfs:
+                utils.set_origin_url(gitdir, checkout)
+                fetch_lfs_binaries(name, checkout)
 
 
 def source_date_epoch(checkout):
diff --git a/ybd/utils.py b/ybd/utils.py
index aa49f0a..ff186c1 100644
--- a/ybd/utils.py
+++ b/ybd/utils.py
@@ -14,6 +14,7 @@
 #
 # =*= License: GPL-2 =*=
 
+import re
 import gzip
 import tarfile
 import contextlib
@@ -25,6 +26,7 @@ from fs.osfs import OSFS
 from fs.multifs import MultiFS
 import calendar
 import app
+from subprocess import check_call, check_output
 
 # The magic number for timestamps: 2011-11-11 11:11:11
 default_magic_timestamp = calendar.timegm([2011, 11, 11, 11, 11, 11])
@@ -447,3 +449,33 @@ def monkeypatch(obj, attr, new_value):
     setattr(obj, attr, new_value)
     yield
     setattr(obj, attr, old_value)
+
+
+def set_origin_url(gitdir, checkout):
+    '''Sets the origin url of a checkout to that of its gitdir.
+
+    git-lfs requires a remote server in order to fetch binaries, so we set the
+    origin url to that of the mirror for lfs enabled checkouts.
+    '''
+    try:
+        with open(os.devnull, 'w') as fnull, app.chdir(gitdir):
+            origin_url = check_output(
+                ['git', 'config', '--get', 'remote.origin.url'], stderr=fnull)
+        with open(os.devnull, 'w') as fnull, app.chdir(checkout):
+            check_call(['git', 'config', 'remote.origin.url', origin_url],
+                       stderr=fnull)
+    except:
+        app.log('UTILS', 'Setting origin url failed for', checkout, exit=True)
+        raise
+
+
+def ref_expects_lfs(gitdir, ref):
+    '''Parses .gitattributes at a ref to determine if git-lfs is required.'''
+    with open(os.devnull, 'w') as fnull, app.chdir(gitdir):
+        blob = check_output(
+            ['git', 'ls-tree', ref, '.gitattributes'], stderr=fnull)
+        if blob:
+            attributes = check_output(
+                ['git', 'cat-file', 'blob', blob.split()[2]], stderr=fnull)
+            return bool(re.search('filter=lfs.*-text', attributes))
+    return False