diff options
Diffstat (limited to 'morphlib')
-rw-r--r-- | morphlib/__init__.py | 2 | ||||
-rw-r--r-- | morphlib/gitdir.py | 131 | ||||
-rw-r--r-- | morphlib/gitdir_tests.py | 66 | ||||
-rw-r--r-- | morphlib/localrepocache.py | 14 | ||||
-rw-r--r-- | morphlib/plugins/branch_and_merge_new_plugin.py | 199 | ||||
-rw-r--r-- | morphlib/plugins/branch_and_merge_plugin.py | 107 | ||||
-rw-r--r-- | morphlib/sysbranchdir.py | 219 | ||||
-rw-r--r-- | morphlib/sysbranchdir_tests.py | 213 | ||||
-rw-r--r-- | morphlib/util.py | 59 | ||||
-rw-r--r-- | morphlib/util_tests.py | 39 | ||||
-rw-r--r-- | morphlib/workspace.py | 34 | ||||
-rw-r--r-- | morphlib/workspace_tests.py | 16 |
12 files changed, 985 insertions, 114 deletions
diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 6b2a1cd7..544dcd09 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -59,6 +59,7 @@ import cachekeycomputer import extractedtarball import fsutils import git +import gitdir import localartifactcache import localrepocache import mountableimage @@ -72,6 +73,7 @@ import source import sourcepool import stagingarea import stopwatch +import sysbranchdir import tempdir import util import workspace diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py new file mode 100644 index 00000000..2bf74437 --- /dev/null +++ b/morphlib/gitdir.py @@ -0,0 +1,131 @@ +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + + +import cliapp + +import morphlib + + +class GitDirectory(object): + + '''Represent a git working tree + .git directory. + + This class represents a directory that is the result of a + "git clone". It includes both the .git subdirectory and + the working tree. It is a thin abstraction, meant to make + it easier to do certain git operations. + + ''' + + def __init__(self, dirname): + self.dirname = dirname + + def _runcmd(self, argv, **kwargs): + '''Run a command at the root of the git directory. + + See cliapp.runcmd for arguments. + + Do NOT use this from outside the class. Add more public + methods for specific git operations instead. + + ''' + + return cliapp.runcmd(argv, cwd=self.dirname, **kwargs) + + def checkout(self, branch_name): # pragma: no cover + '''Check out a git branch.''' + self._runcmd(['git', 'checkout', branch_name]) + + def branch(self, new_branch_name, base_ref): # pragma: no cover + '''Create a git branch based on an existing ref. + + This does not automatically check out the branch. + + base_ref may be None, in which case the current branch is used. + + ''' + + argv = ['git', 'branch', new_branch_name] + if base_ref is not None: + argv.append(base_ref) + self._runcmd(argv) + + def update_remotes(self): # pragma: no cover + '''Update remotes.''' + self._runcmd(['git', 'remote', 'update', '--prune']) + + def update_submodules(self, app): # pragma: no cover + '''Change .gitmodules URLs, and checkout submodules.''' + morphlib.git.update_submodules(app, self.dirname) + + def set_config(self, key, value): + '''Set a git repository configuration variable. + + The key must have at least one period in it: foo.bar for example, + not just foo. The part before the first period is interpreted + by git as a section name. + + ''' + + self._runcmd(['git', 'config', key, value]) + + def get_config(self, key): + '''Return value for a git repository configuration variable.''' + + value = self._runcmd(['git', 'config', key]) + return value.strip() + + def set_remote_fetch_url(self, remote_name, url): + '''Set the fetch URL for a remote.''' + self._runcmd(['git', 'remote', 'set-url', remote_name, url]) + + def get_remote_fetch_url(self, remote_name): + '''Return the fetch URL for a given remote.''' + output = self._runcmd(['git', 'remote', '-v']) + for line in output.splitlines(): + words = line.split() + if (len(words) == 3 and + words[0] == remote_name and + words[2] == '(fetch)'): + return words[1] + return None + + def update_remotes(self): # pragma: no cover + '''Run "git remote update --prune".''' + self._runcmd(['git', 'remote', 'update', '--prune']) + + +def init(dirname): + '''Initialise a new git repository.''' + + gd = GitDirectory(dirname) + gd._runcmd(['git', 'init']) + return gd + + +def clone_from_cached_repo(cached_repo, dirname, ref): # pragma: no cover + '''Clone a CachedRepo into the desired directory. + + The given ref is checked out (or git's default branch is checked out + if ref is None). + + ''' + + cached_repo.clone_checkout(ref, dirname) + return GitDirectory(dirname) + diff --git a/morphlib/gitdir_tests.py b/morphlib/gitdir_tests.py new file mode 100644 index 00000000..2494981a --- /dev/null +++ b/morphlib/gitdir_tests.py @@ -0,0 +1,66 @@ +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + + +import os +import shutil +import tempfile +import unittest + +import morphlib + + +class GitDirectoryTests(unittest.TestCase): + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.dirname = os.path.join(self.tempdir, 'foo') + + def tearDown(self): + shutil.rmtree(self.tempdir) + + def fake_git_clone(self): + os.mkdir(self.dirname) + os.mkdir(os.path.join(self.dirname, '.git')) + + def test_has_dirname_attribute(self): + self.fake_git_clone() + gitdir = morphlib.gitdir.GitDirectory(self.dirname) + self.assertEqual(gitdir.dirname, self.dirname) + + def test_runs_command_in_right_directory(self): + self.fake_git_clone() + gitdir = morphlib.gitdir.GitDirectory(self.dirname) + output = gitdir._runcmd(['pwd']) + self.assertEqual(output.strip(), self.dirname) + + def test_sets_and_gets_configuration(self): + os.mkdir(self.dirname) + gitdir = morphlib.gitdir.init(self.dirname) + gitdir.set_config('foo.bar', 'yoyo') + self.assertEqual(gitdir.get_config('foo.bar'), 'yoyo') + + def test_sets_remote(self): + os.mkdir(self.dirname) + gitdir = morphlib.gitdir.init(self.dirname) + self.assertEqual(gitdir.get_remote_fetch_url('origin'), None) + + gitdir._runcmd(['git', 'remote', 'add', 'origin', 'foobar']) + url = 'git://git.example.com/foo' + gitdir.set_remote_fetch_url('origin', url) + self.assertEqual(gitdir.get_remote_fetch_url('origin'), url) + diff --git a/morphlib/localrepocache.py b/morphlib/localrepocache.py index 465e9f03..aa45cd3d 100644 --- a/morphlib/localrepocache.py +++ b/morphlib/localrepocache.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -285,3 +285,15 @@ class LocalRepoCache(object): self._cached_repo_objects[reponame] = repo return repo raise NotCached(reponame) + + def get_updated_repo(self, reponame): # pragma: no cover + '''Return object representing cached repository, which is updated.''' + + self._app.status(msg='Updating git repository %s in cache' % reponame) + if not self._app.settings['no-git-update']: + cached_repo = self.cache_repo(reponame) + cached_repo.update() + else: + cached_repo = self.get_repo(reponame) + return cached_repo + diff --git a/morphlib/plugins/branch_and_merge_new_plugin.py b/morphlib/plugins/branch_and_merge_new_plugin.py index fb3be920..f52dba6f 100644 --- a/morphlib/plugins/branch_and_merge_new_plugin.py +++ b/morphlib/plugins/branch_and_merge_new_plugin.py @@ -15,6 +15,9 @@ import cliapp +import logging +import os +import shutil import morphlib @@ -26,6 +29,14 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): def enable(self): self.app.add_subcommand('init', self.init, arg_synopsis='[DIR]') self.app.add_subcommand('workspace', self.workspace, arg_synopsis='') + self.app.add_subcommand( + 'checkout', self.checkout, arg_synopsis='REPO BRANCH') + self.app.add_subcommand( + 'branch', self.branch, arg_synopsis='REPO NEW [OLD]') + self.app.add_subcommand( + 'show-system-branch', self.show_system_branch, arg_synopsis='') + self.app.add_subcommand( + 'show-branch-root', self.show_branch_root, arg_synopsis='') def disable(self): pass @@ -68,3 +79,191 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): ws = morphlib.workspace.open('.') self.app.output.write('%s\n' % ws.root) + def checkout(self, args): + '''Check out an existing system branch. + + Command line arguments: + + * `REPO` is the URL to the repository to the root repository of + a system branch. + * `BRANCH` is the name of the system branch. + + This will check out an existing system branch to an existing + workspace. You must create the workspace first. This only checks + out the root repository, not the repositories for individual + components. You need to use `morph edit` to check out those. + + Example: + + cd /src/workspace + morph checkout baserock:baserock/morphs master + + ''' + + if len(args) != 2: + raise cliapp.AppException('morph checkout needs a repo and the ' + 'name of a branch as parameters') + + root_url = args[0] + system_branch = args[1] + + self._require_git_user_config() + + # Open the workspace first thing, so user gets a quick error if + # we're not inside a workspace. + ws = morphlib.workspace.open('.') + + # Make sure the root repository is in the local git repository + # cache, and is up to date. + lrc, rrc = morphlib.util.new_repo_caches(self.app) + cached_repo = lrc.get_updated_repo(root_url) + + # Check the git branch exists. + cached_repo.resolve_ref(system_branch) + + root_dir = ws.get_default_system_branch_directory_name(system_branch) + + try: + # Create the system branch directory. This doesn't yet clone + # the root repository there. + sb = morphlib.sysbranchdir.create( + root_dir, root_url, system_branch) + + gd = sb.clone_cached_repo( + cached_repo, system_branch, system_branch) + gd.update_submodules(self.app) + gd.update_remotes() + except BaseException as e: + # Oops. Clean up. + logging.error('Caught exception: %s' % str(e)) + logging.info('Removing half-finished branch %s' % system_branch) + self._remove_branch_dir_safe(ws.root, root_dir) + raise + + def branch(self, args): + '''Create a new system branch. + + Command line arguments: + + * `REPO` is a repository URL. + * `NEW` is the name of the new system branch. + * `OLD` is the point from which to branch, and defaults to `master`. + + This creates a new system branch. It needs to be run in an + existing workspace (see `morph workspace`). It creates a new + git branch in the clone of the repository in the workspace. The + system branch will not be visible on the git server until you + push your changes to the repository. + + Example: + + cd /src/workspace + morph branch baserock:baserock:morphs jrandom/new-feature + + ''' + + if len(args) not in [2, 3]: + raise cliapp.AppException( + 'morph branch needs name of branch as parameter') + + root_url = args[0] + system_branch = args[1] + base_ref = 'master' if len(args) == 2 else args[2] + origin_base_ref = 'origin/%s' % base_ref + + self._require_git_user_config() + + # Open the workspace first thing, so user gets a quick error if + # we're not inside a workspace. + ws = morphlib.workspace.open('.') + + # Make sure the root repository is in the local git repository + # cache, and is up to date. + lrc, rrc = morphlib.util.new_repo_caches(self.app) + cached_repo = lrc.get_updated_repo(root_url) + + # Make sure the system branch doesn't exist yet. + if cached_repo.ref_exists(system_branch): + raise cliapp.AppException( + 'branch %s already exists in repository %s' % + (system_branch, root_url)) + + # Make sure the base_ref exists. + cached_repo.resolve_ref(base_ref) + + root_dir = ws.get_default_system_branch_directory_name(system_branch) + + try: + # Create the system branch directory. This doesn't yet clone + # the root repository there. + sb = morphlib.sysbranchdir.create( + root_dir, root_url, system_branch) + + gd = sb.clone_cached_repo(cached_repo, system_branch, base_ref) + gd.branch(system_branch, base_ref) + gd.checkout(system_branch) + gd.update_submodules(self.app) + gd.update_remotes() + except BaseException as e: + # Oops. Clean up. + logging.error('Caught exception: %s' % str(e)) + logging.info('Removing half-finished branch %s' % system_branch) + self._remove_branch_dir_safe(ws.root, root_dir) + raise + + def show_system_branch(self, args): + '''Show the name of the current system branch.''' + + ws = morphlib.workspace.open('.') + sb = morphlib.sysbranchdir.open_from_within('.') + self.app.output.write('%s\n' % sb.system_branch_name) + + def show_branch_root(self, args): + '''Show the name of the repository holding the system morphologies. + + This would, for example, write out something like: + + /src/ws/master/baserock:baserock/morphs + + when the master branch of the `baserock:baserock/morphs` + repository is checked out. + + ''' + + ws = morphlib.workspace.open('.') + sb = morphlib.sysbranchdir.open_from_within('.') + self.app.output.write('%s\n' % sb.get_config('branch.root')) + + def _remove_branch_dir_safe(self, workspace_root, system_branch_root): + # This function avoids throwing any exceptions, so it is safe to call + # inside an 'except' block without altering the backtrace. + + def handle_error(function, path, excinfo): + logging.warning ("Error while trying to clean up %s: %s" % + (path, excinfo)) + + shutil.rmtree(system_branch_root, onerror=handle_error) + + # Remove parent directories that are empty too, avoiding exceptions + parent = os.path.dirname(system_branch_root) + while parent != os.path.abspath(workspace_root): + if len(os.listdir(parent)) > 0 or os.path.islink(parent): + break + os.rmdir(parent) + parent = os.path.dirname(parent) + + def _require_git_user_config(self): + '''Warn if the git user.name and user.email variables are not set.''' + + keys = { + 'user.name': 'My Name', + 'user.email': 'me@example.com', + } + + try: + morphlib.git.check_config_set(self.app.runcmd, keys) + except morphlib.git.ConfigNotSetException as e: + self.app.status( + msg="WARNING: %(message)s", + message=str(e), error=True) + diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index d39f815c..62a9f925 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -57,10 +57,6 @@ class BranchAndMergePlugin(cliapp.Plugin): def enable(self): # User-facing commands - self.app.add_subcommand('branch', self.branch, - arg_synopsis='REPO NEW [OLD]') - self.app.add_subcommand('checkout', self.checkout, - arg_synopsis='REPO BRANCH') self.app.add_subcommand('merge', self.merge, arg_synopsis='BRANCH') self.app.add_subcommand('edit', self.edit, @@ -86,12 +82,6 @@ class BranchAndMergePlugin(cliapp.Plugin): self.app.add_subcommand('foreach', self.foreach, arg_synopsis='-- COMMAND [ARGS...]') - # Plumbing commands (FIXME: should be hidden from --help by default) - self.app.add_subcommand('show-system-branch', self.show_system_branch, - arg_synopsis='') - self.app.add_subcommand('show-branch-root', self.show_branch_root, - arg_synopsis='') - def disable(self): pass @@ -583,81 +573,6 @@ class BranchAndMergePlugin(cliapp.Plugin): self.remove_branch_dir_safe(workspace, branch_name) raise - @warns_git_identity - def branch(self, args): - '''Create a new system branch. - - Command line arguments: - - * `REPO` is a repository URL. - * `NEW` is the name of the new system branch. - * `OLD` is the point from which to branch, and defaults to `master`. - - This creates a new system branch. It needs to be run in an - existing workspace (see `morph workspace`). It creates a new - git branch in the clone of the repository in the workspace. The - system branch will not be visible on the git server until you - push your changes to the repository. - - Example: - - cd /src/workspace - morph branch baserock:baserock:morphs jrandom/new-feature - - ''' - - if len(args) not in [2, 3]: - raise cliapp.AppException('morph branch needs name of branch ' - 'as parameter') - - repo = args[0] - new_branch = args[1] - commit = 'master' if len(args) == 2 else args[2] - - self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) - if self.get_cached_repo(repo).ref_exists(new_branch): - raise cliapp.AppException('branch %s already exists in ' - 'repository %s' % (new_branch, repo)) - - # Create the system branch directory. - workspace = self.deduce_workspace() - self._create_branch(workspace, new_branch, repo, commit) - - @warns_git_identity - def checkout(self, args): - '''Check out an existing system branch. - - Command line arguments: - - * `REPO` is the URL to the repository to the root repository of - a system branch. - * `BRANCH` is the name of the system branch. - - This will check out an existing system branch to an existing - workspace. You must create the workspace first. This only checks - out the root repository, not the repositories for individual - components. You need to use `morph edit` to check out those. - - Example: - - cd /src/workspace - morph checkout baserock:baserock/morphs master - - ''' - - if len(args) != 2: - raise cliapp.AppException('morph checkout needs a repo and the ' - 'name of a branch as parameters') - - repo = args[0] - system_branch = args[1] - - self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) - - # Create the system branch directory. - workspace = self.deduce_workspace() - self._create_branch(workspace, system_branch, repo, system_branch) - def checkout_repository(self, branch_dir, repo, ref, parent_ref=None): '''Make a chunk or stratum repository available for a system branch @@ -1997,25 +1912,3 @@ class BranchAndMergePlugin(cliapp.Plugin): raise cliapp.AppException( 'Command failed at repo %s: %s' % (repo, ' '.join(args))) - def show_system_branch(self, args): - '''Show the name of the current system branch.''' - - branch, dirname = self.deduce_system_branch() - self.app.output.write('%s\n' % branch) - - def show_branch_root(self, args): - '''Show the name of the repository holding the system morphologies. - - This would, for example, write out something like: - - /src/ws/master/baserock:baserock/morphs - - when the master branch of the `baserock:baserock/morphs` - repository is checked out. - - ''' - - workspace = self.deduce_workspace() - system_branch, branch_dir = self.deduce_system_branch() - branch_root = self.get_branch_config(branch_dir, 'branch.root') - self.app.output.write('%s\n' % branch_root) diff --git a/morphlib/sysbranchdir.py b/morphlib/sysbranchdir.py new file mode 100644 index 00000000..e4af53cf --- /dev/null +++ b/morphlib/sysbranchdir.py @@ -0,0 +1,219 @@ +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + + +import cliapp +import os +import urlparse +import uuid + +import morphlib + + +class SystemBranchDirectoryAlreadyExists(morphlib.Error): + + def __init__(self, root_directory): + self.msg = ( + "%s: File exists" % + root_directory) + + +class NotInSystemBranch(morphlib.Error): + + def __init__(self, dirname): + self.msg = ( + "Can't find the system branch directory.\n" + "Morph must be built and deployed within " + "the system branch checkout.") + + +class SystemBranchDirectory(object): + + '''A directory containing a checked out system branch.''' + + def __init__(self, + root_directory, root_repository_url, system_branch_name): + self.root_directory = root_directory + self.root_repository_url = root_repository_url + self.system_branch_name = system_branch_name + + @property + def _magic_path(self): + return os.path.join(self.root_directory, '.morph-system-branch') + + @property + def _config_path(self): + return os.path.join(self._magic_path, 'config') + + def set_config(self, key, value): + '''Set a configuration key/value pair.''' + cliapp.runcmd(['git', 'config', '-f', self._config_path, key, value]) + + def get_config(self, key): + '''Get a configuration value for a given key.''' + value = cliapp.runcmd(['git', 'config', '-f', self._config_path, key]) + return value.strip() + + def get_git_directory_name(self, repo_url): + '''Return directory pathname for a given git repository. + + If the URL is a real one (not aliased), the schema and leading // + are removed from it, as is a .git suffix. + + ''' + + # Parse the URL. If the path component is absolute, we assume + # it's a real URL; otherwise, an aliased URL. + parts = urlparse.urlparse(repo_url) + + if os.path.isabs(parts.path): + # Remove .git suffix, if any. + path = parts.path + if path.endswith('.git'): + path = path[:-len('.git')] + + # Add the domain name etc (netloc). Ignore any other parts. + # Note that we _know_ the path starts with a slash, so we avoid + # adding one here. + relative = '%s%s' % (parts.netloc, path) + else: + relative = repo_url + + # Remove anyleading slashes, or os.path.join below will only + # use the relative part (since it's absolute, not relative). + while relative.startswith('/'): + relative = relative[1:] + + return os.path.join(self.root_directory, relative) + + def clone_cached_repo(self, cached_repo, git_branch_name, checkout_ref): + '''Clone a cached git repository into the system branch directory. + + The cloned repository will NOT have the system branch's git branch + checked out: instead, checkout_ref is checked out (this is for + backwards compatibility with older implementation of "morph + branch"; it may change later). The system branch's git branch + is NOT created: the caller will need to do that. Submodules are + NOT checked out. + + The "origin" remote will be set to follow the cached repository's + upstream. Remotes are not updated. + + ''' + + # Do the clone. + dirname = self.get_git_directory_name(cached_repo.original_name) + gd = morphlib.gitdir.clone_from_cached_repo( + cached_repo, dirname, checkout_ref) + + # Remember the repo name we cloned from in order to be able + # to identify the repo again later using the same name, even + # if the user happens to rename the directory. + gd.set_config('morph.repository', cached_repo.original_name) + + # Create a UUID for the clone. We will use this for naming + # temporary refs, e.g. for building. + gd.set_config('morph.uuid', uuid.uuid4().hex) + + # Configure the "origin" remote to use the upstream git repository, + # and not the locally cached copy. + resolver = morphlib.repoaliasresolver.RepoAliasResolver( + cached_repo.app.settings['repo-alias']) + gd.set_remote_fetch_url('origin', resolver.pull_url(cached_repo.url)) + gd.set_config( + 'url.%s.pushInsteadOf' % + resolver.push_url(cached_repo.original_name), + resolver.pull_url(cached_repo.url)) + + return gd + + def list_git_directories(self): + '''List all git directories in a system branch directory. + + The list will contain zero or more GitDirectory objects. + + ''' + + gitdirs = [] + for dirname, subdirs, filenames in os.walk(self.root_directory): + if os.path.isdir(os.path.join(dirname, '.git')): + del subdirs[:] + gitdirs.append(morphlib.gitdir.GitDirectory(dirname)) + + return gitdirs + + +def create(root_directory, root_repository_url, system_branch_name): + '''Create a new system branch directory on disk. + + Return a SystemBranchDirectory object that represents the directory. + + The directory MUST NOT exist already. If it does, + SystemBranchDirectoryAlreadyExists is raised. + + Note that this does NOT check out the root repository, or do any + other git cloning. + + ''' + + if os.path.exists(root_directory): + raise SystemBranchDirectoryAlreadyExists(root_directory) + + magic_dir = os.path.join(root_directory, '.morph-system-branch') + os.makedirs(root_directory) + os.mkdir(magic_dir) + + sb = SystemBranchDirectory( + root_directory, root_repository_url, system_branch_name) + sb.set_config('branch.name', system_branch_name) + sb.set_config('branch.root', root_repository_url) + sb.set_config('branch.uuid', uuid.uuid4().hex) + + return sb + + +def open(root_directory): + '''Open an existing system branch directory.''' + + # Ugly hack follows. + sb = SystemBranchDirectory(root_directory, None, None) + root_repository_url = sb.get_config('branch.root') + system_branch_name = sb.get_config('branch.name') + + return SystemBranchDirectory( + root_directory, root_repository_url, system_branch_name) + + +def open_from_within(dirname): + '''Open a system branch directory, given any directory. + + The directory can be within the system branch root directory, + or it can be a parent, in some cases. If each parent on the + path from dirname to the system branch root directory has no + siblings, this function will find it. + + ''' + + root_directory = morphlib.util.find_root( + dirname, '.morph-system-branch') + if root_directory is None: + root_directory = morphlib.util.find_leaf( + dirname, '.morph-system-branch') + if root_directory is None: + raise NotInSystemBranch(dirname) + return morphlib.sysbranchdir.open(root_directory) + diff --git a/morphlib/sysbranchdir_tests.py b/morphlib/sysbranchdir_tests.py new file mode 100644 index 00000000..8e62791f --- /dev/null +++ b/morphlib/sysbranchdir_tests.py @@ -0,0 +1,213 @@ +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + + +import cliapp +import os +import shutil +import tempfile +import unittest + +import morphlib + + +class SystemBranchDirectoryTests(unittest.TestCase): + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.root_directory = os.path.join(self.tempdir, 'rootdir') + self.root_repository_url = 'test:morphs' + self.system_branch_name = 'foo/bar' + + def tearDown(self): + shutil.rmtree(self.tempdir) + + def create_fake_cached_repo(self): + + class FakeCachedRepo(object): + + def __init__(self, url, path): + self.app = self + self.settings = { + 'repo-alias': [], + } + self.original_name = url + self.url = 'git://blahlbah/blah/blahblahblah.git' + self.path = path + + os.mkdir(self.path) + cliapp.runcmd(['git', 'init', self.path]) + with open(os.path.join(self.path, 'filename'), 'w') as f: + f.write('this is a file\n') + cliapp.runcmd(['git', 'add', 'filename'], cwd=self.path) + cliapp.runcmd( + ['git', 'commit', '-m', 'initial'], cwd=self.path) + + def clone_checkout(self, ref, target_dir): + cliapp.runcmd( + ['git', 'clone', '-b', ref, self.path, target_dir]) + + subdir = tempfile.mkdtemp(dir=self.tempdir) + path = os.path.join(subdir, 'foo') + return FakeCachedRepo(self.root_repository_url, path) + + def test_creates_system_branch_directory(self): + sb = morphlib.sysbranchdir.create( + self.root_directory, + self.root_repository_url, + self.system_branch_name) + self.assertEqual(sb.root_directory, self.root_directory) + self.assertEqual(sb.root_repository_url, self.root_repository_url) + self.assertEqual(sb.system_branch_name, self.system_branch_name) + + magic_dir = os.path.join(self.root_directory, '.morph-system-branch') + self.assertTrue(os.path.isdir(self.root_directory)) + self.assertTrue(os.path.isdir(magic_dir)) + self.assertTrue(os.path.isfile(os.path.join(magic_dir, 'config'))) + self.assertEqual( + sb.get_config('branch.root'), self.root_repository_url) + self.assertEqual( + sb.get_config('branch.name'), self.system_branch_name) + self.assertTrue(sb.get_config('branch.uuid')) + + def test_opens_system_branch_directory(self): + morphlib.sysbranchdir.create( + self.root_directory, + self.root_repository_url, + self.system_branch_name) + sb = morphlib.sysbranchdir.open(self.root_directory) + self.assertEqual(sb.root_directory, self.root_directory) + self.assertEqual(sb.root_repository_url, self.root_repository_url) + self.assertEqual(sb.system_branch_name, self.system_branch_name) + + def test_opens_system_branch_directory_from_a_subdirectory(self): + morphlib.sysbranchdir.create( + self.root_directory, + self.root_repository_url, + self.system_branch_name) + subdir = os.path.join(self.root_directory, 'a', 'b', 'c') + os.makedirs(subdir) + sb = morphlib.sysbranchdir.open_from_within(subdir) + self.assertEqual(sb.root_directory, self.root_directory) + self.assertEqual(sb.root_repository_url, self.root_repository_url) + self.assertEqual(sb.system_branch_name, self.system_branch_name) + + def test_fails_opening_system_branch_directory_when_none_exists(self): + self.assertRaises( + morphlib.sysbranchdir.NotInSystemBranch, + morphlib.sysbranchdir.open_from_within, + self.tempdir) + + def test_opens_system_branch_directory_when_it_is_the_only_child(self): + deep_root = os.path.join(self.tempdir, 'a', 'b', 'c') + morphlib.sysbranchdir.create( + deep_root, + self.root_repository_url, + self.system_branch_name) + sb = morphlib.sysbranchdir.open(deep_root) + self.assertEqual(sb.root_directory, deep_root) + self.assertEqual(sb.root_repository_url, self.root_repository_url) + self.assertEqual(sb.system_branch_name, self.system_branch_name) + + def test_fails_to_create_if_directory_already_exists(self): + os.mkdir(self.root_directory) + self.assertRaises( + morphlib.sysbranchdir.SystemBranchDirectoryAlreadyExists, + morphlib.sysbranchdir.create, + self.root_directory, + self.root_repository_url, + self.system_branch_name) + + def test_sets_and_gets_configuration_values(self): + sb = morphlib.sysbranchdir.create( + self.root_directory, + self.root_repository_url, + self.system_branch_name) + sb.set_config('foo.key', 'foovalue') + + sb2 = morphlib.sysbranchdir.open(self.root_directory) + self.assertEqual(sb2.get_config('foo.key'), 'foovalue') + + def test_reports_correct_name_for_git_directory_from_aliases_url(self): + sb = morphlib.sysbranchdir.create( + self.root_directory, + self.root_repository_url, + self.system_branch_name) + self.assertEqual( + sb.get_git_directory_name('baserock:baserock/morph'), + os.path.join(self.root_directory, 'baserock:baserock/morph')) + + def test_reports_correct_name_for_git_directory_from_real_url(self): + stripped = 'git.baserock.org/baserock/baserock/morph' + url = 'git://%s.git' % stripped + sb = morphlib.sysbranchdir.create( + self.root_directory, + url, + self.system_branch_name) + self.assertEqual( + sb.get_git_directory_name(url), + os.path.join(self.root_directory, stripped)) + + def test_reports_correct_name_for_git_directory_from_file_url(self): + stripped = 'foobar/morphs' + url = 'file:///%s.git' % stripped + sb = morphlib.sysbranchdir.create( + self.root_directory, + url, + self.system_branch_name) + self.assertEqual( + sb.get_git_directory_name(url), + os.path.join(self.root_directory, stripped)) + + def test_clones_git_repository(self): + + sb = morphlib.sysbranchdir.create( + self.root_directory, + self.root_repository_url, + self.system_branch_name) + + cached_repo = self.create_fake_cached_repo() + gd = sb.clone_cached_repo( + cached_repo, self.system_branch_name, 'master') + + self.assertEqual( + gd.dirname, + sb.get_git_directory_name(cached_repo.original_name)) + + def test_lists_git_directories(self): + + def fake_git_clone(dirname, url, branch): + os.mkdir(dirname) + subdir = os.path.join(dirname, '.git') + os.mkdir(subdir) + + sb = morphlib.sysbranchdir.create( + self.root_directory, + self.root_repository_url, + self.system_branch_name) + + sb._git_clone = fake_git_clone + + cached_repo = self.create_fake_cached_repo() + sb.clone_cached_repo(cached_repo, 'branch1', 'master') + + gd_list = sb.list_git_directories() + self.assertEqual(len(gd_list), 1) + self.assertEqual( + gd_list[0].dirname, + sb.get_git_directory_name(cached_repo.original_name)) + diff --git a/morphlib/util.py b/morphlib/util.py index a9c22217..ead0bafe 100644 --- a/morphlib/util.py +++ b/morphlib/util.py @@ -197,28 +197,28 @@ def copyfileobj(inputfp, outputfp, blocksize=1024*1024): # pragma: no cover while 1: inbuf = inputfp.read(blocksize) if not inbuf: break - if not buf: + if not buf: buf = inbuf else: buf += inbuf - + # Combine "short" reads if (len(buf) < blocksize): continue - + buflen = len(buf) if buf == "\x00" * buflen: outputfp.seek(buflen, os.SEEK_CUR) buf = None # flag sparse=True, that we seek()ed, but have not written yet # The filesize is wrong until we write - sparse = True + sparse = True else: outputfp.write(buf) buf = None # We wrote, so clear sparse. sparse = False - + if buf: outputfp.write(buf) elif sparse: @@ -244,7 +244,7 @@ def on_same_filesystem(path_a, path_b): # pragma: no cover def unify_space_requirements(tmp_path, tmp_min_size, cache_path, cache_min_size): # pragma: no cover """Adjust minimum sizes when paths share a disk. - + Given pairs of path and minimum size, return the minimum sizes such that when the paths are on the same disk, the sizes are added together. @@ -280,3 +280,50 @@ def check_disk_available(tmp_path, tmp_min_size, 'space or reduce the disk space required by the ' 'tempdir-min-space and cachedir-min-space ' 'configuration options.') + + + + +def find_root(dirname, subdir_name): + '''Find parent of a directory, at or above a given directory. + + The sought-after directory is indicated by the existence of a + subdirectory of the indicated name. For example, dirname might + be the current working directory of the process, and subdir_name + might be ".morph"; then the returned value would be the Morph + workspace root directory, which has a subdirectory called + ".morph". + + Return path to desired directory, or None if not found. + + ''' + + dirname = os.path.normpath(os.path.abspath(dirname)) + while not os.path.isdir(os.path.join(dirname, subdir_name)): + if dirname == '/': + return None + dirname = os.path.dirname(dirname) + return dirname + + +def find_leaf(dirname, subdir_name): + '''This is like find_root, except it looks towards leaves. + + It only looks in a subdirectory if it is the only subdirectory. + If there are no subdirectories, or more than one, fail. + (Subdirectories whose name starts with a dot are ignored for this.) + + ''' + + while True: + if os.path.exists(os.path.join(dirname, subdir_name)): + return dirname + pathnames = [ + os.path.join(dirname, x) + for x in os.listdir(dirname) + if not x.startswith('.')] + subdirs = [x for x in pathnames if os.path.isdir(x)] + if len(subdirs) != 1: + return None + dirname = subdirs[0] + diff --git a/morphlib/util_tests.py b/morphlib/util_tests.py index 89fe184e..eaff0821 100644 --- a/morphlib/util_tests.py +++ b/morphlib/util_tests.py @@ -14,6 +14,9 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import os +import shutil +import tempfile import unittest import morphlib @@ -48,3 +51,39 @@ class MakeConcurrencyTests(unittest.TestCase): def test_returns_6_for_4_cores(self): self.assertEqual(morphlib.util.make_concurrency(cores=4), 6) + + +class FindParentOfTests(unittest.TestCase): + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + os.makedirs(os.path.join(self.tempdir, 'a', 'b', 'c')) + self.a = os.path.join(self.tempdir, 'a') + self.b = os.path.join(self.tempdir, 'a', 'b') + self.c = os.path.join(self.tempdir, 'a', 'b', 'c') + + def tearDown(self): + shutil.rmtree(self.tempdir) + + def test_find_root_finds_starting_directory(self): + os.mkdir(os.path.join(self.a, '.magic')) + self.assertEqual(morphlib.util.find_root(self.a, '.magic'), self.a) + + def test_find_root_finds_ancestor(self): + os.mkdir(os.path.join(self.a, '.magic')) + self.assertEqual(morphlib.util.find_root(self.c, '.magic'), self.a) + + def test_find_root_returns_none_if_not_found(self): + self.assertEqual(morphlib.util.find_root(self.c, '.magic'), None) + + def test_find_leaf_finds_starting_directory(self): + os.mkdir(os.path.join(self.a, '.magic')) + self.assertEqual(morphlib.util.find_leaf(self.a, '.magic'), self.a) + + def test_find_leaf_finds_child(self): + os.mkdir(os.path.join(self.c, '.magic')) + self.assertEqual(morphlib.util.find_leaf(self.a, '.magic'), self.c) + + def test_find_leaf_returns_none_if_not_found(self): + self.assertEqual(morphlib.util.find_leaf(self.a, '.magic'), None) + diff --git a/morphlib/workspace.py b/morphlib/workspace.py index 3a2269c8..93f699e6 100644 --- a/morphlib/workspace.py +++ b/morphlib/workspace.py @@ -53,6 +53,40 @@ class Workspace(object): def __init__(self, root_directory): self.root = root_directory + def get_default_system_branch_directory_name(self, system_branch_name): + '''Determine directory where a system branch would be checked out. + + Return the fully qualified pathname to the directory where + a system branch would be checked out. The directory may or may + not exist already. + + If the system branch is checked out, but into a directory of + a different name (which is allowed), that is ignored: this method + only computed the default name. + + ''' + + return os.path.join(self.root, system_branch_name) + + def create_system_branch_directory(self, + root_repository_url, system_branch_name): + '''Create a directory for a system branch. + + Return a SystemBranchDirectory object that represents the + directory. The directory must not already exist. The directory + gets created and initialised (the .morph-system-branch/config + file gets created and populated). The root repository of the + system branch does NOT get checked out, the caller needs to + do that. + + ''' + + dirname = self.get_default_system_branch_directory_name( + system_branch_name) + sb = morphlib.sysbranchdir.create( + dirname, root_repository_url, system_branch_name) + return sb + def open(dirname): '''Open an existing workspace. diff --git a/morphlib/workspace_tests.py b/morphlib/workspace_tests.py index 7837481a..83b5e54f 100644 --- a/morphlib/workspace_tests.py +++ b/morphlib/workspace_tests.py @@ -83,3 +83,19 @@ class WorkspaceTests(unittest.TestCase): morphlib.workspace.NotInWorkspace, morphlib.workspace.open, self.tempdir) + def test_invents_appropriate_name_for_system_branch_directory(self): + self.create_it() + ws = morphlib.workspace.open(self.workspace_dir) + branch = 'foo/bar' + self.assertEqual( + ws.get_default_system_branch_directory_name(branch), + os.path.join(self.workspace_dir, branch)) + + def test_creates_system_branch_directory(self): + self.create_it() + ws = morphlib.workspace.open(self.workspace_dir) + url = 'test:morphs' + branch = 'my/new/thing' + sb = ws.create_system_branch_directory(url, branch) + self.assertTrue(type(sb), morphlib.sysbranchdir.SystemBranchDirectory) + |