diff options
author | Adam Coldrick <adam.coldrick@codethink.co.uk> | 2014-03-20 14:57:12 +0000 |
---|---|---|
committer | Adam Coldrick <adam.coldrick@codethink.co.uk> | 2014-03-20 14:57:12 +0000 |
commit | b585c5bdc2d9da874618f60a5fe8e3f16160be15 (patch) | |
tree | 5a32f8dac5c792366b2735805b02c8c7daf48c23 /morphlib | |
parent | 86cb3fb50a8d04911f5191eb80edb2295350e120 (diff) | |
parent | 7e317e1b118e3adddf863c7dbdecfbe09c45fcf1 (diff) | |
download | morph-b585c5bdc2d9da874618f60a5fe8e3f16160be15.tar.gz |
Merge branch 'adamcoldrick/ingest-binaries-v5-rebase'
Author: Adam Coldrick <adam.coldrick@codethink.co.uk>
Reviewed by:
* Richard Maw <richard.maw@codethink.co.uk>
* Lars Wirzenius <lars.wirzenius@codethink.co.uk>
* Richard Ipsum <richard.ipsum@codethink.co.uk>
Diffstat (limited to 'morphlib')
-rw-r--r-- | morphlib/git.py | 10 | ||||
-rw-r--r-- | morphlib/gitdir.py | 37 | ||||
-rw-r--r-- | morphlib/plugins/add_binary_plugin.py | 110 | ||||
-rw-r--r-- | morphlib/plugins/branch_and_merge_new_plugin.py | 10 | ||||
-rw-r--r-- | morphlib/plugins/push_pull_plugin.py | 93 |
5 files changed, 257 insertions, 3 deletions
diff --git a/morphlib/git.py b/morphlib/git.py index 27146206..ccd06323 100644 --- a/morphlib/git.py +++ b/morphlib/git.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -279,6 +279,10 @@ def copy_repository(runcmd, repo, destdir, is_mirror=True): def checkout_ref(runcmd, gitdir, ref): '''Checks out a specific ref/SHA1 in a git working tree.''' runcmd(['git', 'checkout', ref], cwd=gitdir) + gd = morphlib.gitdir.GitDirectory(gitdir) + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() def index_has_changes(runcmd, gitdir): @@ -308,6 +312,10 @@ def clone_into(runcmd, srcpath, targetpath, ref=None): runcmd(['git', 'checkout', ref], cwd=targetpath) else: runcmd(['git', 'clone', '-b', ref, srcpath, targetpath]) + gd = morphlib.gitdir.GitDirectory(targetpath) + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() def is_valid_sha1(ref): '''Checks whether a string is a valid SHA1.''' diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py index 15079231..06fcba6f 100644 --- a/morphlib/gitdir.py +++ b/morphlib/gitdir.py @@ -317,6 +317,14 @@ class Remote(object): self._parse_push_output(out), err) return self._parse_push_output(out) + def pull(self, branch=None): # pragma: no cover + if branch: + repo = self.get_fetch_url() + ret = self.gd._runcmd(['git', 'pull', repo, branch]) + else: + ret = self.gd._runcmd(['git', 'pull']) + return ret + class GitDirectory(object): @@ -330,7 +338,11 @@ class GitDirectory(object): ''' def __init__(self, dirname): - self.dirname = dirname + self.dirname = morphlib.util.find_root(dirname, '.git') + # if we are in a bare repo, self.dirname will now be None + # so we just use the provided dirname + if not self.dirname: + self.dirname = dirname def _runcmd(self, argv, **kwargs): '''Run a command at the root of the git directory. @@ -350,6 +362,9 @@ class GitDirectory(object): def checkout(self, branch_name): # pragma: no cover '''Check out a git branch.''' self._runcmd(['git', 'checkout', branch_name]) + if self.has_fat(): + self.fat_init() + self.fat_pull() def branch(self, new_branch_name, base_ref): # pragma: no cover '''Create a git branch based on an existing ref. @@ -616,12 +631,30 @@ class GitDirectory(object): ['git', 'describe', '--always', '--dirty=-unreproducible']) return version.strip() + def fat_init(self): # pragma: no cover + return self._runcmd(['git', 'fat', 'init']) + + def fat_push(self): # pragma: no cover + return self._runcmd(['git', 'fat', 'push']) + + def fat_pull(self): # pragma: no cover + return self._runcmd(['git', 'fat', 'pull']) + + def has_fat(self): # pragma: no cover + return '.gitfat' in self.list_files() + + def join_path(self, path): # pragma: no cover + return os.path.join(self.dirname, path) + + def get_relpath(self, path): # pragma: no cover + return os.path.relpath(path, self.dirname) + def init(dirname): '''Initialise a new git repository.''' + cliapp.runcmd(['git', 'init'], cwd=dirname) gd = GitDirectory(dirname) - gd._runcmd(['git', 'init']) return gd diff --git a/morphlib/plugins/add_binary_plugin.py b/morphlib/plugins/add_binary_plugin.py new file mode 100644 index 00000000..1edae0e8 --- /dev/null +++ b/morphlib/plugins/add_binary_plugin.py @@ -0,0 +1,110 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import logging +import os +import urlparse + +import morphlib + + +class AddBinaryPlugin(cliapp.Plugin): + + '''Add a subcommand for dealing with large binary files.''' + + def enable(self): + self.app.add_subcommand( + 'add-binary', self.add_binary, arg_synopsis='FILENAME...') + + def disable(self): + pass + + def add_binary(self, binaries): + '''Add a binary file to the current repository. + + Command line argument: + + * `FILENAME...` is the binaries to be added to the repository. + + This checks for the existence of a .gitfat file in the repository. If + there is one then a line is added to .gitattributes telling it that + the given binary should be handled by git-fat. If there is no .gitfat + file then it is created, with the rsync remote pointing at the correct + directory on the Trove host. A line is then added to .gitattributes to + say that the given binary should be handled by git-fat. + + Example: + + morph add-binary big_binary.tar.gz + + ''' + if not binaries: + raise morphlib.Error('add-binary must get at least one argument') + + gd = morphlib.gitdir.GitDirectory(os.getcwd()) + gd.fat_init() + if not gd.has_fat(): + self._make_gitfat(gd) + self._handle_binaries(binaries, gd) + logging.info('Staged binaries for commit') + + def _handle_binaries(self, binaries, gd): + '''Add a filter for the given file, and then add it to the repo.''' + # begin by ensuring all paths given are relative to the root directory + files = [gd.get_relpath(os.path.realpath(binary)) + for binary in binaries] + + # now add any files that aren't already mentioned in .gitattributes to + # the file so that git fat knows what to do + attr_path = gd.join_path('.gitattributes') + if '.gitattributes' in gd.list_files(): + with open(attr_path, 'r') as attributes: + current = set(f.split()[0] for f in attributes) + else: + current = set() + to_add = set(files) - current + + # if we don't need to change .gitattributes then we can just do + # `git add <binaries>` + if not to_add: + gd.get_index().add_files_from_working_tree(files) + return + + with open(attr_path, 'a') as attributes: + for path in to_add: + attributes.write('%s filter=fat -crlf\n' % path) + + # we changed .gitattributes, so need to stage it for committing + files.append(attr_path) + gd.get_index().add_files_from_working_tree(files) + + def _make_gitfat(self, gd): + '''Make .gitfat point to the rsync directory for the repo.''' + remote = gd.get_remote('origin') + if not remote.get_push_url(): + raise Exception( + 'Remote `origin` does not have a push URL defined.') + url = urlparse.urlparse(remote.get_push_url()) + if url.scheme != 'ssh': + raise Exception( + 'Push URL for `origin` is not an SSH URL: %s' % url.geturl()) + fat_store = '%s:%s' % (url.netloc, url.path) + fat_path = gd.join_path('.gitfat') + with open(fat_path, 'w+') as gitfat: + gitfat.write('[rsync]\n') + gitfat.write('remote = %s' % fat_store) + gd.get_index().add_files_from_working_tree([fat_path]) diff --git a/morphlib/plugins/branch_and_merge_new_plugin.py b/morphlib/plugins/branch_and_merge_new_plugin.py index 8c8a98e9..51cba401 100644 --- a/morphlib/plugins/branch_and_merge_new_plugin.py +++ b/morphlib/plugins/branch_and_merge_new_plugin.py @@ -190,6 +190,10 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): with self._initializing_system_branch( ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd): + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() + if not self._checkout_has_systems(gd): raise BranchRootHasNoSystemsError(root_url, base_ref) @@ -250,6 +254,9 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): gd.branch(system_branch, base_ref) gd.checkout(system_branch) + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() if not self._checkout_has_systems(gd): raise BranchRootHasNoSystemsError(root_url, base_ref) @@ -480,6 +487,9 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): gd.checkout(sb.system_branch_name) gd.update_submodules(self.app) gd.update_remotes() + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() # Change the refs to the chunk. if chunk_ref != sb.system_branch_name: diff --git a/morphlib/plugins/push_pull_plugin.py b/morphlib/plugins/push_pull_plugin.py new file mode 100644 index 00000000..843de1a6 --- /dev/null +++ b/morphlib/plugins/push_pull_plugin.py @@ -0,0 +1,93 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import cliapp +import logging +import os + +import morphlib + + +class PushPullPlugin(cliapp.Plugin): + + '''Add subcommands to wrap the git push and pull commands.''' + + def enable(self): + self.app.add_subcommand( + 'push', self.push, arg_synopsis='REPO TARGET') + self.app.add_subcommand('pull', self.pull, arg_synopsis='[REMOTE]') + + def disable(self): + pass + + def push(self, args): + '''Push a branch to a remote repository. + + Command line arguments: + + * `REPO` is the repository to push your changes to. + + * `TARGET` is the branch to push to the repository. + + This is a wrapper for the `git push` command. It also deals with + pushing any binary files that have been added using git-fat. + + Example: + + morph push origin jrandom/new-feature + + ''' + if len(args) != 2: + raise morphlib.Error('push must get exactly two arguments') + + gd = morphlib.gitdir.GitDirectory(os.getcwd()) + remote, branch = args + rs = morphlib.gitdir.RefSpec(branch) + gd.get_remote(remote).push(rs) + if gd.has_fat(): + gd.fat_init() + gd.fat_push() + + def pull(self, args): + '''Pull changes to the current branch from a repository. + + Command line arguments: + + * `REMOTE` is the remote branch to pull from. By default this is the + branch being tracked by your current git branch (ie origin/master + for branch master) + + This is a wrapper for the `git pull` command. It also deals with + pulling any binary files that have been added to the repository using + git-fat. + + Example: + + morph pull + + ''' + if len(args) > 1: + raise morphlib.Error('pull takes at most one argument') + + gd = morphlib.gitdir.GitDirectory(os.getcwd()) + remote = gd.get_remote('origin') + if args: + branch = args[0] + remote.pull(branch) + else: + remote.pull() + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() |