From 73e22af6a9c83f9e9e1d79f4018f4562bf5ae1b2 Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Fri, 14 Mar 2014 12:58:32 +0000 Subject: Add functionality for doing git commands in a directory These commands are: * git fat * git pull They are required for the morph add-binary and push/pull plugins. Also make sure that GitDirectory is working in the root directory of the specified git repository, and add some helper functions for handling paths of files in the working tree. --- morphlib/gitdir.py | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py index 15079231..15c0ee9f 100644 --- a/morphlib/gitdir.py +++ b/morphlib/gitdir.py @@ -317,6 +317,14 @@ class Remote(object): self._parse_push_output(out), err) return self._parse_push_output(out) + def pull(self, branch=None): # pragma: no cover + if branch: + repo = self.get_fetch_url() + ret = self.gd._runcmd(['git', 'pull', repo, branch]) + else: + ret = self.gd._runcmd(['git', 'pull']) + return ret + class GitDirectory(object): @@ -330,7 +338,11 @@ class GitDirectory(object): ''' def __init__(self, dirname): - self.dirname = dirname + self.dirname = morphlib.util.find_root(dirname, '.git') + # if we are in a bare repo, self.dirname will now be None + # so we just use the provided dirname + if not self.dirname: + self.dirname = dirname def _runcmd(self, argv, **kwargs): '''Run a command at the root of the git directory. @@ -616,12 +628,30 @@ class GitDirectory(object): ['git', 'describe', '--always', '--dirty=-unreproducible']) return version.strip() + def fat_init(self): # pragma: no cover + return self._runcmd(['git', 'fat', 'init']) + + def fat_push(self): # pragma: no cover + return self._runcmd(['git', 'fat', 'push']) + + def fat_pull(self): # pragma: no cover + return self._runcmd(['git', 'fat', 'pull']) + + def has_fat(self): # pragma: no cover + return '.gitfat' in self.list_files() + + def join_path(self, path): # pragma: no cover + return os.path.join(self.dirname, path) + + def get_relpath(self, path): # pragma: no cover + return os.path.relpath(path, self.dirname) + def init(dirname): '''Initialise a new git repository.''' + cliapp.runcmd(['git', 'init'], cwd=dirname) gd = GitDirectory(dirname) - gd._runcmd(['git', 'init']) return gd -- cgit v1.2.1 From cbc117355a6f549a14cae57dac43fa6f432c1849 Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Fri, 14 Mar 2014 13:01:45 +0000 Subject: Make existing morph commands use git-fat When cloning a repository, the files stored using git-fat need to be pulled. This situation occurs in `morph branch`, `morph edit`, and `morph checkout`. --- morphlib/git.py | 10 +++++++++- morphlib/gitdir.py | 3 +++ morphlib/plugins/branch_and_merge_new_plugin.py | 10 ++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/morphlib/git.py b/morphlib/git.py index 27146206..ccd06323 100644 --- a/morphlib/git.py +++ b/morphlib/git.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -279,6 +279,10 @@ def copy_repository(runcmd, repo, destdir, is_mirror=True): def checkout_ref(runcmd, gitdir, ref): '''Checks out a specific ref/SHA1 in a git working tree.''' runcmd(['git', 'checkout', ref], cwd=gitdir) + gd = morphlib.gitdir.GitDirectory(gitdir) + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() def index_has_changes(runcmd, gitdir): @@ -308,6 +312,10 @@ def clone_into(runcmd, srcpath, targetpath, ref=None): runcmd(['git', 'checkout', ref], cwd=targetpath) else: runcmd(['git', 'clone', '-b', ref, srcpath, targetpath]) + gd = morphlib.gitdir.GitDirectory(targetpath) + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() def is_valid_sha1(ref): '''Checks whether a string is a valid SHA1.''' diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py index 15c0ee9f..06fcba6f 100644 --- a/morphlib/gitdir.py +++ b/morphlib/gitdir.py @@ -362,6 +362,9 @@ class GitDirectory(object): def checkout(self, branch_name): # pragma: no cover '''Check out a git branch.''' self._runcmd(['git', 'checkout', branch_name]) + if self.has_fat(): + self.fat_init() + self.fat_pull() def branch(self, new_branch_name, base_ref): # pragma: no cover '''Create a git branch based on an existing ref. diff --git a/morphlib/plugins/branch_and_merge_new_plugin.py b/morphlib/plugins/branch_and_merge_new_plugin.py index 8c8a98e9..51cba401 100644 --- a/morphlib/plugins/branch_and_merge_new_plugin.py +++ b/morphlib/plugins/branch_and_merge_new_plugin.py @@ -190,6 +190,10 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): with self._initializing_system_branch( ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd): + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() + if not self._checkout_has_systems(gd): raise BranchRootHasNoSystemsError(root_url, base_ref) @@ -250,6 +254,9 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): gd.branch(system_branch, base_ref) gd.checkout(system_branch) + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() if not self._checkout_has_systems(gd): raise BranchRootHasNoSystemsError(root_url, base_ref) @@ -480,6 +487,9 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): gd.checkout(sb.system_branch_name) gd.update_submodules(self.app) gd.update_remotes() + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() # Change the refs to the chunk. if chunk_ref != sb.system_branch_name: -- cgit v1.2.1 From ff07b21cfa01d12e2a5d2c883ec47a9ee6415e0f Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Fri, 14 Mar 2014 13:03:18 +0000 Subject: Implement morph add-binary using git-fat to store large files Add a plugin which implements the morph add-binary command. This command is used to add large files to a git repository. It sets up the files needed to use git-fat, and then runs `git add` with git-fat initiated. --- morphlib/plugins/add_binary_plugin.py | 110 ++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 morphlib/plugins/add_binary_plugin.py diff --git a/morphlib/plugins/add_binary_plugin.py b/morphlib/plugins/add_binary_plugin.py new file mode 100644 index 00000000..1edae0e8 --- /dev/null +++ b/morphlib/plugins/add_binary_plugin.py @@ -0,0 +1,110 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import logging +import os +import urlparse + +import morphlib + + +class AddBinaryPlugin(cliapp.Plugin): + + '''Add a subcommand for dealing with large binary files.''' + + def enable(self): + self.app.add_subcommand( + 'add-binary', self.add_binary, arg_synopsis='FILENAME...') + + def disable(self): + pass + + def add_binary(self, binaries): + '''Add a binary file to the current repository. + + Command line argument: + + * `FILENAME...` is the binaries to be added to the repository. + + This checks for the existence of a .gitfat file in the repository. If + there is one then a line is added to .gitattributes telling it that + the given binary should be handled by git-fat. If there is no .gitfat + file then it is created, with the rsync remote pointing at the correct + directory on the Trove host. A line is then added to .gitattributes to + say that the given binary should be handled by git-fat. + + Example: + + morph add-binary big_binary.tar.gz + + ''' + if not binaries: + raise morphlib.Error('add-binary must get at least one argument') + + gd = morphlib.gitdir.GitDirectory(os.getcwd()) + gd.fat_init() + if not gd.has_fat(): + self._make_gitfat(gd) + self._handle_binaries(binaries, gd) + logging.info('Staged binaries for commit') + + def _handle_binaries(self, binaries, gd): + '''Add a filter for the given file, and then add it to the repo.''' + # begin by ensuring all paths given are relative to the root directory + files = [gd.get_relpath(os.path.realpath(binary)) + for binary in binaries] + + # now add any files that aren't already mentioned in .gitattributes to + # the file so that git fat knows what to do + attr_path = gd.join_path('.gitattributes') + if '.gitattributes' in gd.list_files(): + with open(attr_path, 'r') as attributes: + current = set(f.split()[0] for f in attributes) + else: + current = set() + to_add = set(files) - current + + # if we don't need to change .gitattributes then we can just do + # `git add ` + if not to_add: + gd.get_index().add_files_from_working_tree(files) + return + + with open(attr_path, 'a') as attributes: + for path in to_add: + attributes.write('%s filter=fat -crlf\n' % path) + + # we changed .gitattributes, so need to stage it for committing + files.append(attr_path) + gd.get_index().add_files_from_working_tree(files) + + def _make_gitfat(self, gd): + '''Make .gitfat point to the rsync directory for the repo.''' + remote = gd.get_remote('origin') + if not remote.get_push_url(): + raise Exception( + 'Remote `origin` does not have a push URL defined.') + url = urlparse.urlparse(remote.get_push_url()) + if url.scheme != 'ssh': + raise Exception( + 'Push URL for `origin` is not an SSH URL: %s' % url.geturl()) + fat_store = '%s:%s' % (url.netloc, url.path) + fat_path = gd.join_path('.gitfat') + with open(fat_path, 'w+') as gitfat: + gitfat.write('[rsync]\n') + gitfat.write('remote = %s' % fat_store) + gd.get_index().add_files_from_working_tree([fat_path]) -- cgit v1.2.1 From 7e317e1b118e3adddf863c7dbdecfbe09c45fcf1 Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Fri, 14 Mar 2014 13:05:40 +0000 Subject: Implement morph push and morph pull Add a plugin to implement both `morph push` and `morph pull`. These commands are wrappers around the corresponding git commands push and pull, which also implement the functionality of pushing and pulling large files provided by git-fat. For example, running `morph pull` will pull any commits from the remote branch not on your local branch, and then pull any large files from the separate git-fat/rsync store on the Trove. --- morphlib/plugins/push_pull_plugin.py | 93 ++++++++++++++++++++++++++++++++++++ without-test-modules | 2 + 2 files changed, 95 insertions(+) create mode 100644 morphlib/plugins/push_pull_plugin.py diff --git a/morphlib/plugins/push_pull_plugin.py b/morphlib/plugins/push_pull_plugin.py new file mode 100644 index 00000000..843de1a6 --- /dev/null +++ b/morphlib/plugins/push_pull_plugin.py @@ -0,0 +1,93 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import cliapp +import logging +import os + +import morphlib + + +class PushPullPlugin(cliapp.Plugin): + + '''Add subcommands to wrap the git push and pull commands.''' + + def enable(self): + self.app.add_subcommand( + 'push', self.push, arg_synopsis='REPO TARGET') + self.app.add_subcommand('pull', self.pull, arg_synopsis='[REMOTE]') + + def disable(self): + pass + + def push(self, args): + '''Push a branch to a remote repository. + + Command line arguments: + + * `REPO` is the repository to push your changes to. + + * `TARGET` is the branch to push to the repository. + + This is a wrapper for the `git push` command. It also deals with + pushing any binary files that have been added using git-fat. + + Example: + + morph push origin jrandom/new-feature + + ''' + if len(args) != 2: + raise morphlib.Error('push must get exactly two arguments') + + gd = morphlib.gitdir.GitDirectory(os.getcwd()) + remote, branch = args + rs = morphlib.gitdir.RefSpec(branch) + gd.get_remote(remote).push(rs) + if gd.has_fat(): + gd.fat_init() + gd.fat_push() + + def pull(self, args): + '''Pull changes to the current branch from a repository. + + Command line arguments: + + * `REMOTE` is the remote branch to pull from. By default this is the + branch being tracked by your current git branch (ie origin/master + for branch master) + + This is a wrapper for the `git pull` command. It also deals with + pulling any binary files that have been added to the repository using + git-fat. + + Example: + + morph pull + + ''' + if len(args) > 1: + raise morphlib.Error('pull takes at most one argument') + + gd = morphlib.gitdir.GitDirectory(os.getcwd()) + remote = gd.get_remote('origin') + if args: + branch = args[0] + remote.pull(branch) + else: + remote.pull() + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() diff --git a/without-test-modules b/without-test-modules index 1f5bc872..29f88f17 100644 --- a/without-test-modules +++ b/without-test-modules @@ -29,5 +29,7 @@ morphlib/plugins/trovectl_plugin.py morphlib/plugins/gc_plugin.py morphlib/plugins/branch_and_merge_new_plugin.py morphlib/plugins/print_architecture_plugin.py +morphlib/plugins/add_binary_plugin.py +morphlib/plugins/push_pull_plugin.py # Not unit tested, since it needs a full system branch morphlib/buildbranch.py -- cgit v1.2.1