Merge branch 'adamcoldrick/ingest-binaries-v5-rebase'

Author: Adam Coldrick <adam.coldrick@codethink.co.uk>

Reviewed by:
  * Richard Maw <richard.maw@codethink.co.uk>
  * Lars Wirzenius <lars.wirzenius@codethink.co.uk>
  * Richard Ipsum <richard.ipsum@codethink.co.uk>

6 files changed, 259 insertions, 3 deletions

diff --git a/morphlib/git.py b/morphlib/git.py
index 27146206..ccd06323 100644
--- a/morphlib/git.py
+++ b/morphlib/git.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2011-2013  Codethink Limited
+# Copyright (C) 2011-2014  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -279,6 +279,10 @@ def copy_repository(runcmd, repo, destdir, is_mirror=True):
 def checkout_ref(runcmd, gitdir, ref):
     '''Checks out a specific ref/SHA1 in a git working tree.'''
     runcmd(['git', 'checkout', ref], cwd=gitdir)
+    gd = morphlib.gitdir.GitDirectory(gitdir)
+    if gd.has_fat():
+        gd.fat_init()
+        gd.fat_pull()
 
 
 def index_has_changes(runcmd, gitdir):
@@ -308,6 +312,10 @@ def clone_into(runcmd, srcpath, targetpath, ref=None):
         runcmd(['git', 'checkout', ref], cwd=targetpath)
     else:
         runcmd(['git', 'clone', '-b', ref, srcpath, targetpath])
+    gd = morphlib.gitdir.GitDirectory(targetpath)
+    if gd.has_fat():
+        gd.fat_init()
+        gd.fat_pull()
 
 def is_valid_sha1(ref):
     '''Checks whether a string is a valid SHA1.'''
diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py
index 15079231..06fcba6f 100644
--- a/morphlib/gitdir.py
+++ b/morphlib/gitdir.py
@@ -317,6 +317,14 @@ class Remote(object):
                                    self._parse_push_output(out), err)
         return self._parse_push_output(out)
 
+    def pull(self, branch=None): # pragma: no cover
+        if branch:
+            repo = self.get_fetch_url()
+            ret = self.gd._runcmd(['git', 'pull', repo, branch])
+        else:
+            ret = self.gd._runcmd(['git', 'pull'])
+        return ret
+
 
 class GitDirectory(object):
 
@@ -330,7 +338,11 @@ class GitDirectory(object):
     '''
 
     def __init__(self, dirname):
-        self.dirname = dirname
+        self.dirname = morphlib.util.find_root(dirname, '.git')
+        # if we are in a bare repo, self.dirname will now be None
+        # so we just use the provided dirname
+        if not self.dirname:
+            self.dirname = dirname
 
     def _runcmd(self, argv, **kwargs):
         '''Run a command at the root of the git directory.
@@ -350,6 +362,9 @@ class GitDirectory(object):
     def checkout(self, branch_name): # pragma: no cover
         '''Check out a git branch.'''
         self._runcmd(['git', 'checkout', branch_name])
+        if self.has_fat():
+            self.fat_init()
+            self.fat_pull()
 
     def branch(self, new_branch_name, base_ref): # pragma: no cover
         '''Create a git branch based on an existing ref.
@@ -616,12 +631,30 @@ class GitDirectory(object):
             ['git', 'describe', '--always', '--dirty=-unreproducible'])
         return version.strip()
 
+    def fat_init(self): # pragma: no cover
+        return self._runcmd(['git', 'fat', 'init'])
+
+    def fat_push(self): # pragma: no cover
+        return self._runcmd(['git', 'fat', 'push'])
+
+    def fat_pull(self): # pragma: no cover
+        return self._runcmd(['git', 'fat', 'pull'])
+
+    def has_fat(self): # pragma: no cover
+        return '.gitfat' in self.list_files()
+
+    def join_path(self, path): # pragma: no cover
+        return os.path.join(self.dirname, path)
+
+    def get_relpath(self, path): # pragma: no cover
+        return os.path.relpath(path, self.dirname)
+
 
 def init(dirname):
     '''Initialise a new git repository.'''
 
+    cliapp.runcmd(['git', 'init'], cwd=dirname)
     gd = GitDirectory(dirname)
-    gd._runcmd(['git', 'init'])
     return gd
 
 
diff --git a/morphlib/plugins/add_binary_plugin.py b/morphlib/plugins/add_binary_plugin.py
new file mode 100644
index 00000000..1edae0e8
--- /dev/null
+++ b/morphlib/plugins/add_binary_plugin.py
@@ -0,0 +1,110 @@
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import cliapp
+import logging
+import os
+import urlparse
+
+import morphlib
+
+
+class AddBinaryPlugin(cliapp.Plugin):
+
+    '''Add a subcommand for dealing with large binary files.'''
+
+    def enable(self):
+        self.app.add_subcommand(
+            'add-binary', self.add_binary, arg_synopsis='FILENAME...')
+
+    def disable(self):
+        pass
+
+    def add_binary(self, binaries):
+        '''Add a binary file to the current repository.
+
+        Command line argument:
+
+        * `FILENAME...` is the binaries to be added to the repository.
+
+        This checks for the existence of a .gitfat file in the repository. If
+        there is one then a line is added to .gitattributes telling it that
+        the given binary should be handled by git-fat. If there is no .gitfat
+        file then it is created, with the rsync remote pointing at the correct
+        directory on the Trove host. A line is then added to .gitattributes to
+        say that the given binary should be handled by git-fat.
+
+        Example:
+
+            morph add-binary big_binary.tar.gz
+
+        '''
+        if not binaries:
+            raise morphlib.Error('add-binary must get at least one argument')
+
+        gd = morphlib.gitdir.GitDirectory(os.getcwd())
+        gd.fat_init()
+        if not gd.has_fat():
+            self._make_gitfat(gd)
+        self._handle_binaries(binaries, gd)
+        logging.info('Staged binaries for commit')
+
+    def _handle_binaries(self, binaries, gd):
+        '''Add a filter for the given file, and then add it to the repo.'''
+        # begin by ensuring all paths given are relative to the root directory
+        files = [gd.get_relpath(os.path.realpath(binary))
+                 for binary in binaries]
+
+        # now add any files that aren't already mentioned in .gitattributes to
+        # the file so that git fat knows what to do
+        attr_path = gd.join_path('.gitattributes')
+        if '.gitattributes' in gd.list_files():
+            with open(attr_path, 'r') as attributes:
+                current = set(f.split()[0] for f in attributes)
+        else:
+            current = set()
+        to_add = set(files) - current
+
+        # if we don't need to change .gitattributes then we can just do
+        # `git add <binaries>`
+        if not to_add:
+            gd.get_index().add_files_from_working_tree(files)
+            return
+
+        with open(attr_path, 'a') as attributes:
+            for path in to_add:
+                attributes.write('%s filter=fat -crlf\n' % path)
+
+        # we changed .gitattributes, so need to stage it for committing
+        files.append(attr_path)
+        gd.get_index().add_files_from_working_tree(files)
+
+    def _make_gitfat(self, gd):
+        '''Make .gitfat point to the rsync directory for the repo.'''
+        remote = gd.get_remote('origin')
+        if not remote.get_push_url():
+            raise Exception(
+                'Remote `origin` does not have a push URL defined.')
+        url = urlparse.urlparse(remote.get_push_url())
+        if url.scheme != 'ssh':
+            raise Exception(
+                'Push URL for `origin` is not an SSH URL: %s' % url.geturl())
+        fat_store = '%s:%s' % (url.netloc, url.path)
+        fat_path = gd.join_path('.gitfat')
+        with open(fat_path, 'w+') as gitfat:
+            gitfat.write('[rsync]\n')
+            gitfat.write('remote = %s' % fat_store)
+        gd.get_index().add_files_from_working_tree([fat_path])
diff --git a/morphlib/plugins/branch_and_merge_new_plugin.py b/morphlib/plugins/branch_and_merge_new_plugin.py
index 8c8a98e9..51cba401 100644
--- a/morphlib/plugins/branch_and_merge_new_plugin.py
+++ b/morphlib/plugins/branch_and_merge_new_plugin.py
@@ -190,6 +190,10 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin):
         with self._initializing_system_branch(
             ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd):
 
+            if gd.has_fat():
+                gd.fat_init()
+                gd.fat_pull()
+
             if not self._checkout_has_systems(gd):
                 raise BranchRootHasNoSystemsError(root_url, base_ref)
 
@@ -250,6 +254,9 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin):
 
             gd.branch(system_branch, base_ref)
             gd.checkout(system_branch)
+            if gd.has_fat():
+                gd.fat_init()
+                gd.fat_pull()
 
             if not self._checkout_has_systems(gd):
                 raise BranchRootHasNoSystemsError(root_url, base_ref)
@@ -480,6 +487,9 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin):
                     gd.checkout(sb.system_branch_name)
                 gd.update_submodules(self.app)
                 gd.update_remotes()
+                if gd.has_fat():
+                    gd.fat_init()
+                    gd.fat_pull()
 
                 # Change the refs to the chunk.
                 if chunk_ref != sb.system_branch_name:
diff --git a/morphlib/plugins/push_pull_plugin.py b/morphlib/plugins/push_pull_plugin.py
new file mode 100644
index 00000000..843de1a6
--- /dev/null
+++ b/morphlib/plugins/push_pull_plugin.py
@@ -0,0 +1,93 @@
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import cliapp
+import logging
+import os
+
+import morphlib
+
+
+class PushPullPlugin(cliapp.Plugin):
+
+    '''Add subcommands to wrap the git push and pull commands.'''
+
+    def enable(self):
+        self.app.add_subcommand(
+            'push', self.push, arg_synopsis='REPO TARGET')
+        self.app.add_subcommand('pull', self.pull, arg_synopsis='[REMOTE]')
+
+    def disable(self):
+        pass
+
+    def push(self, args):
+        '''Push a branch to a remote repository.
+
+        Command line arguments:
+
+        * `REPO` is the repository to push your changes to.
+
+        * `TARGET` is the branch to push to the repository.
+
+        This is a wrapper for the `git push` command. It also deals with
+        pushing any binary files that have been added using git-fat.
+
+        Example:
+
+            morph push origin jrandom/new-feature
+
+        '''
+        if len(args) != 2:
+            raise morphlib.Error('push must get exactly two arguments')
+
+        gd = morphlib.gitdir.GitDirectory(os.getcwd())
+        remote, branch = args
+        rs = morphlib.gitdir.RefSpec(branch)
+        gd.get_remote(remote).push(rs)
+        if gd.has_fat():
+            gd.fat_init()
+            gd.fat_push()
+
+    def pull(self, args):
+        '''Pull changes to the current branch from a repository.
+
+        Command line arguments:
+
+        * `REMOTE` is the remote branch to pull from. By default this is the
+          branch being tracked by your current git branch (ie origin/master
+          for branch master)
+
+        This is a wrapper for the `git pull` command. It also deals with
+        pulling any binary files that have been added to the repository using
+        git-fat.
+
+        Example:
+
+            morph pull
+
+        '''
+        if len(args) > 1:
+            raise morphlib.Error('pull takes at most one argument')
+
+        gd = morphlib.gitdir.GitDirectory(os.getcwd())
+        remote = gd.get_remote('origin')
+        if args:
+            branch = args[0]
+            remote.pull(branch)
+        else:
+            remote.pull()
+        if gd.has_fat():
+            gd.fat_init()
+            gd.fat_pull()
diff --git a/without-test-modules b/without-test-modules
index 1f5bc872..29f88f17 100644
--- a/without-test-modules
+++ b/without-test-modules
@@ -29,5 +29,7 @@ morphlib/plugins/trovectl_plugin.py
 morphlib/plugins/gc_plugin.py
 morphlib/plugins/branch_and_merge_new_plugin.py
 morphlib/plugins/print_architecture_plugin.py
+morphlib/plugins/add_binary_plugin.py
+morphlib/plugins/push_pull_plugin.py
 # Not unit tested, since it needs a full system branch
 morphlib/buildbranch.py