Merge branch 'liw/refactor-checkout-etc-v2'

Reviewed-by: various

12 files changed, 985 insertions, 114 deletions

diff --git a/morphlib/__init__.py b/morphlib/__init__.py
index 6b2a1cd7..544dcd09 100644
--- a/morphlib/__init__.py
+++ b/morphlib/__init__.py
@@ -59,6 +59,7 @@ import cachekeycomputer
 import extractedtarball
 import fsutils
 import git
+import gitdir
 import localartifactcache
 import localrepocache
 import mountableimage
@@ -72,6 +73,7 @@ import source
 import sourcepool
 import stagingarea
 import stopwatch
+import sysbranchdir
 import tempdir
 import util
 import workspace
diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py
new file mode 100644
index 00000000..2bf74437
--- /dev/null
+++ b/morphlib/gitdir.py
@@ -0,0 +1,131 @@
+# Copyright (C) 2013  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# =*= License: GPL-2 =*=
+
+
+import cliapp
+
+import morphlib
+
+
+class GitDirectory(object):
+
+    '''Represent a git working tree + .git directory.
+
+    This class represents a directory that is the result of a
+    "git clone". It includes both the .git subdirectory and
+    the working tree. It is a thin abstraction, meant to make
+    it easier to do certain git operations.
+
+    '''
+
+    def __init__(self, dirname):
+        self.dirname = dirname
+
+    def _runcmd(self, argv, **kwargs):
+        '''Run a command at the root of the git directory.
+
+        See cliapp.runcmd for arguments.
+
+        Do NOT use this from outside the class. Add more public
+        methods for specific git operations instead.
+
+        '''
+
+        return cliapp.runcmd(argv, cwd=self.dirname, **kwargs)
+
+    def checkout(self, branch_name): # pragma: no cover
+        '''Check out a git branch.'''
+        self._runcmd(['git', 'checkout', branch_name])
+
+    def branch(self, new_branch_name, base_ref): # pragma: no cover
+        '''Create a git branch based on an existing ref.
+
+        This does not automatically check out the branch.
+
+        base_ref may be None, in which case the current branch is used.
+
+        '''
+
+        argv = ['git', 'branch', new_branch_name]
+        if base_ref is not None:
+            argv.append(base_ref)
+        self._runcmd(argv)
+
+    def update_remotes(self): # pragma: no cover
+        '''Update remotes.'''
+        self._runcmd(['git', 'remote', 'update', '--prune'])
+
+    def update_submodules(self, app): # pragma: no cover
+        '''Change .gitmodules URLs, and checkout submodules.'''
+        morphlib.git.update_submodules(app, self.dirname)
+
+    def set_config(self, key, value):
+        '''Set a git repository configuration variable.
+
+        The key must have at least one period in it: foo.bar for example,
+        not just foo. The part before the first period is interpreted
+        by git as a section name.
+
+        '''
+
+        self._runcmd(['git', 'config', key, value])
+
+    def get_config(self, key):
+        '''Return value for a git repository configuration variable.'''
+
+        value = self._runcmd(['git', 'config', key])
+        return value.strip()
+
+    def set_remote_fetch_url(self, remote_name, url):
+        '''Set the fetch URL for a remote.'''
+        self._runcmd(['git', 'remote', 'set-url', remote_name, url])
+
+    def get_remote_fetch_url(self, remote_name):
+        '''Return the fetch URL for a given remote.'''
+        output = self._runcmd(['git', 'remote', '-v'])
+        for line in output.splitlines():
+            words = line.split()
+            if (len(words) == 3 and
+                words[0] == remote_name and
+                words[2] == '(fetch)'):
+                return words[1]
+        return None
+
+    def update_remotes(self): # pragma: no cover
+        '''Run "git remote update --prune".'''
+        self._runcmd(['git', 'remote', 'update', '--prune'])
+
+
+def init(dirname):
+    '''Initialise a new git repository.'''
+
+    gd = GitDirectory(dirname)
+    gd._runcmd(['git', 'init'])
+    return gd
+
+
+def clone_from_cached_repo(cached_repo, dirname, ref): # pragma: no cover
+    '''Clone a CachedRepo into the desired directory.
+
+    The given ref is checked out (or git's default branch is checked out
+    if ref is None).
+
+    '''
+
+    cached_repo.clone_checkout(ref, dirname)
+    return GitDirectory(dirname)
+
diff --git a/morphlib/gitdir_tests.py b/morphlib/gitdir_tests.py
new file mode 100644
index 00000000..2494981a
--- /dev/null
+++ b/morphlib/gitdir_tests.py
@@ -0,0 +1,66 @@
+# Copyright (C) 2013  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# =*= License: GPL-2 =*=
+
+
+import os
+import shutil
+import tempfile
+import unittest
+
+import morphlib
+
+
+class GitDirectoryTests(unittest.TestCase):
+
+    def setUp(self):
+        self.tempdir = tempfile.mkdtemp()
+        self.dirname = os.path.join(self.tempdir, 'foo')
+
+    def tearDown(self):
+        shutil.rmtree(self.tempdir)
+
+    def fake_git_clone(self):
+        os.mkdir(self.dirname)
+        os.mkdir(os.path.join(self.dirname, '.git'))
+
+    def test_has_dirname_attribute(self):
+        self.fake_git_clone()
+        gitdir = morphlib.gitdir.GitDirectory(self.dirname)
+        self.assertEqual(gitdir.dirname, self.dirname)
+
+    def test_runs_command_in_right_directory(self):
+        self.fake_git_clone()
+        gitdir = morphlib.gitdir.GitDirectory(self.dirname)
+        output = gitdir._runcmd(['pwd'])
+        self.assertEqual(output.strip(), self.dirname)
+
+    def test_sets_and_gets_configuration(self):
+        os.mkdir(self.dirname)
+        gitdir = morphlib.gitdir.init(self.dirname)
+        gitdir.set_config('foo.bar', 'yoyo')
+        self.assertEqual(gitdir.get_config('foo.bar'), 'yoyo')
+
+    def test_sets_remote(self):
+        os.mkdir(self.dirname)
+        gitdir = morphlib.gitdir.init(self.dirname)
+        self.assertEqual(gitdir.get_remote_fetch_url('origin'), None)
+
+        gitdir._runcmd(['git', 'remote', 'add', 'origin', 'foobar'])
+        url = 'git://git.example.com/foo'
+        gitdir.set_remote_fetch_url('origin', url)
+        self.assertEqual(gitdir.get_remote_fetch_url('origin'), url)
+
diff --git a/morphlib/localrepocache.py b/morphlib/localrepocache.py
index 465e9f03..aa45cd3d 100644
--- a/morphlib/localrepocache.py
+++ b/morphlib/localrepocache.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012  Codethink Limited
+# Copyright (C) 2012-2013  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -285,3 +285,15 @@ class LocalRepoCache(object):
                 self._cached_repo_objects[reponame] = repo
                 return repo
         raise NotCached(reponame)
+
+    def get_updated_repo(self, reponame): # pragma: no cover
+        '''Return object representing cached repository, which is updated.'''
+
+        self._app.status(msg='Updating git repository %s in cache' % reponame)
+        if not self._app.settings['no-git-update']:
+            cached_repo = self.cache_repo(reponame)
+            cached_repo.update()
+        else:
+            cached_repo = self.get_repo(reponame)
+        return cached_repo
+
diff --git a/morphlib/plugins/branch_and_merge_new_plugin.py b/morphlib/plugins/branch_and_merge_new_plugin.py
index fb3be920..f52dba6f 100644
--- a/morphlib/plugins/branch_and_merge_new_plugin.py
+++ b/morphlib/plugins/branch_and_merge_new_plugin.py
@@ -15,6 +15,9 @@
 
 
 import cliapp
+import logging
+import os
+import shutil
 
 import morphlib
 
@@ -26,6 +29,14 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin):
     def enable(self):
         self.app.add_subcommand('init', self.init, arg_synopsis='[DIR]')
         self.app.add_subcommand('workspace', self.workspace, arg_synopsis='')
+        self.app.add_subcommand(
+            'checkout', self.checkout, arg_synopsis='REPO BRANCH')
+        self.app.add_subcommand(
+            'branch', self.branch, arg_synopsis='REPO NEW [OLD]')
+        self.app.add_subcommand(
+            'show-system-branch', self.show_system_branch, arg_synopsis='')
+        self.app.add_subcommand(
+            'show-branch-root', self.show_branch_root, arg_synopsis='')
 
     def disable(self):
         pass
@@ -68,3 +79,191 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin):
         ws = morphlib.workspace.open('.')
         self.app.output.write('%s\n' % ws.root)
 
+    def checkout(self, args):
+        '''Check out an existing system branch.
+
+        Command line arguments:
+
+        * `REPO` is the URL to the repository to the root repository of
+          a system branch.
+        * `BRANCH` is the name of the system branch.
+
+        This will check out an existing system branch to an existing
+        workspace.  You must create the workspace first. This only checks
+        out the root repository, not the repositories for individual
+        components. You need to use `morph edit` to check out those.
+
+        Example:
+
+            cd /src/workspace
+            morph checkout baserock:baserock/morphs master
+
+        '''
+
+        if len(args) != 2:
+            raise cliapp.AppException('morph checkout needs a repo and the '
+                                      'name of a branch as parameters')
+
+        root_url = args[0]
+        system_branch = args[1]
+
+        self._require_git_user_config()
+
+        # Open the workspace first thing, so user gets a quick error if
+        # we're not inside a workspace.
+        ws = morphlib.workspace.open('.')
+
+        # Make sure the root repository is in the local git repository
+        # cache, and is up to date.
+        lrc, rrc = morphlib.util.new_repo_caches(self.app)
+        cached_repo = lrc.get_updated_repo(root_url)
+
+        # Check the git branch exists.
+        cached_repo.resolve_ref(system_branch)
+
+        root_dir = ws.get_default_system_branch_directory_name(system_branch)
+
+        try:
+            # Create the system branch directory. This doesn't yet clone
+            # the root repository there.
+            sb = morphlib.sysbranchdir.create(
+                root_dir, root_url, system_branch)
+
+            gd = sb.clone_cached_repo(
+                cached_repo, system_branch, system_branch)
+            gd.update_submodules(self.app)
+            gd.update_remotes()
+        except BaseException as e:
+            # Oops. Clean up.
+            logging.error('Caught exception: %s' % str(e))
+            logging.info('Removing half-finished branch %s' % system_branch)
+            self._remove_branch_dir_safe(ws.root, root_dir)
+            raise
+
+    def branch(self, args):
+        '''Create a new system branch.
+
+        Command line arguments:
+
+        * `REPO` is a repository URL.
+        * `NEW` is the name of the new system branch.
+        * `OLD` is the point from which to branch, and defaults to `master`.
+
+        This creates a new system branch. It needs to be run in an
+        existing workspace (see `morph workspace`). It creates a new
+        git branch in the clone of the repository in the workspace. The
+        system branch will not be visible on the git server until you
+        push your changes to the repository.
+
+        Example:
+
+            cd /src/workspace
+            morph branch baserock:baserock:morphs jrandom/new-feature
+
+        '''
+
+        if len(args) not in [2, 3]:
+            raise cliapp.AppException(
+                'morph branch needs name of branch as parameter')
+
+        root_url = args[0]
+        system_branch = args[1]
+        base_ref = 'master' if len(args) == 2 else args[2]
+        origin_base_ref = 'origin/%s' % base_ref
+
+        self._require_git_user_config()
+
+        # Open the workspace first thing, so user gets a quick error if
+        # we're not inside a workspace.
+        ws = morphlib.workspace.open('.')
+
+        # Make sure the root repository is in the local git repository
+        # cache, and is up to date.
+        lrc, rrc = morphlib.util.new_repo_caches(self.app)
+        cached_repo = lrc.get_updated_repo(root_url)
+
+        # Make sure the system branch doesn't exist yet.
+        if cached_repo.ref_exists(system_branch):
+            raise cliapp.AppException(
+                'branch %s already exists in repository %s' %
+                (system_branch, root_url))
+
+        # Make sure the base_ref exists.
+        cached_repo.resolve_ref(base_ref)
+
+        root_dir = ws.get_default_system_branch_directory_name(system_branch)
+
+        try:
+            # Create the system branch directory. This doesn't yet clone
+            # the root repository there.
+            sb = morphlib.sysbranchdir.create(
+                root_dir, root_url, system_branch)
+
+            gd = sb.clone_cached_repo(cached_repo, system_branch, base_ref)
+            gd.branch(system_branch, base_ref)
+            gd.checkout(system_branch)
+            gd.update_submodules(self.app)
+            gd.update_remotes()
+        except BaseException as e:
+            # Oops. Clean up.
+            logging.error('Caught exception: %s' % str(e))
+            logging.info('Removing half-finished branch %s' % system_branch)
+            self._remove_branch_dir_safe(ws.root, root_dir)
+            raise
+
+    def show_system_branch(self, args):
+        '''Show the name of the current system branch.'''
+
+        ws = morphlib.workspace.open('.')
+        sb = morphlib.sysbranchdir.open_from_within('.')
+        self.app.output.write('%s\n' % sb.system_branch_name)
+
+    def show_branch_root(self, args):
+        '''Show the name of the repository holding the system morphologies.
+
+        This would, for example, write out something like:
+
+            /src/ws/master/baserock:baserock/morphs
+
+        when the master branch of the `baserock:baserock/morphs`
+        repository is checked out.
+
+        '''
+
+        ws = morphlib.workspace.open('.')
+        sb = morphlib.sysbranchdir.open_from_within('.')
+        self.app.output.write('%s\n' % sb.get_config('branch.root'))
+
+    def _remove_branch_dir_safe(self, workspace_root, system_branch_root):
+        # This function avoids throwing any exceptions, so it is safe to call
+        # inside an 'except' block without altering the backtrace.
+
+        def handle_error(function, path, excinfo):
+            logging.warning ("Error while trying to clean up %s: %s" %
+                             (path, excinfo))
+
+        shutil.rmtree(system_branch_root, onerror=handle_error)
+
+        # Remove parent directories that are empty too, avoiding exceptions
+        parent = os.path.dirname(system_branch_root)
+        while parent != os.path.abspath(workspace_root):
+            if len(os.listdir(parent)) > 0 or os.path.islink(parent):
+                break
+            os.rmdir(parent)
+            parent = os.path.dirname(parent)
+
+    def _require_git_user_config(self):
+        '''Warn if the git user.name and user.email variables are not set.'''
+
+        keys = {
+            'user.name': 'My Name',
+            'user.email': 'me@example.com',
+        }
+
+        try:
+            morphlib.git.check_config_set(self.app.runcmd, keys)
+        except morphlib.git.ConfigNotSetException as e:
+            self.app.status(
+                msg="WARNING: %(message)s",
+                message=str(e), error=True)
+
diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py
index d39f815c..62a9f925 100644
--- a/morphlib/plugins/branch_and_merge_plugin.py
+++ b/morphlib/plugins/branch_and_merge_plugin.py
@@ -57,10 +57,6 @@ class BranchAndMergePlugin(cliapp.Plugin):
 
     def enable(self):
         # User-facing commands
-        self.app.add_subcommand('branch', self.branch,
-                                arg_synopsis='REPO NEW [OLD]')
-        self.app.add_subcommand('checkout', self.checkout,
-                                arg_synopsis='REPO BRANCH')
         self.app.add_subcommand('merge', self.merge,
                                 arg_synopsis='BRANCH')
         self.app.add_subcommand('edit', self.edit,
@@ -86,12 +82,6 @@ class BranchAndMergePlugin(cliapp.Plugin):
         self.app.add_subcommand('foreach', self.foreach,
                                 arg_synopsis='-- COMMAND [ARGS...]')
 
-        # Plumbing commands (FIXME: should be hidden from --help by default)
-        self.app.add_subcommand('show-system-branch', self.show_system_branch,
-                                arg_synopsis='')
-        self.app.add_subcommand('show-branch-root', self.show_branch_root,
-                                arg_synopsis='')
-
     def disable(self):
         pass
 
@@ -583,81 +573,6 @@ class BranchAndMergePlugin(cliapp.Plugin):
             self.remove_branch_dir_safe(workspace, branch_name)
             raise
 
-    @warns_git_identity
-    def branch(self, args):
-        '''Create a new system branch.
-
-        Command line arguments:
-
-        * `REPO` is a repository URL.
-        * `NEW` is the name of the new system branch.
-        * `OLD` is the point from which to branch, and defaults to `master`.
-
-        This creates a new system branch. It needs to be run in an
-        existing workspace (see `morph workspace`). It creates a new
-        git branch in the clone of the repository in the workspace. The
-        system branch will not be visible on the git server until you
-        push your changes to the repository.
-
-        Example:
-
-            cd /src/workspace
-            morph branch baserock:baserock:morphs jrandom/new-feature
-
-        '''
-
-        if len(args) not in [2, 3]:
-            raise cliapp.AppException('morph branch needs name of branch '
-                                      'as parameter')
-
-        repo = args[0]
-        new_branch = args[1]
-        commit = 'master' if len(args) == 2 else args[2]
-
-        self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app)
-        if self.get_cached_repo(repo).ref_exists(new_branch):
-            raise cliapp.AppException('branch %s already exists in '
-                                      'repository %s' % (new_branch, repo))
-
-        # Create the system branch directory.
-        workspace = self.deduce_workspace()
-        self._create_branch(workspace, new_branch, repo, commit)
-
-    @warns_git_identity
-    def checkout(self, args):
-        '''Check out an existing system branch.
-
-        Command line arguments:
-
-        * `REPO` is the URL to the repository to the root repository of
-          a system branch.
-        * `BRANCH` is the name of the system branch.
-
-        This will check out an existing system branch to an existing
-        workspace.  You must create the workspace first. This only checks
-        out the root repository, not the repositories for individual
-        components. You need to use `morph edit` to check out those.
-
-        Example:
-
-            cd /src/workspace
-            morph checkout baserock:baserock/morphs master
-
-        '''
-
-        if len(args) != 2:
-            raise cliapp.AppException('morph checkout needs a repo and the '
-                                      'name of a branch as parameters')
-
-        repo = args[0]
-        system_branch = args[1]
-
-        self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app)
-
-        # Create the system branch directory.
-        workspace = self.deduce_workspace()
-        self._create_branch(workspace, system_branch, repo, system_branch)
-
     def checkout_repository(self, branch_dir, repo, ref, parent_ref=None):
         '''Make a chunk or stratum repository available for a system branch
 
@@ -1997,25 +1912,3 @@ class BranchAndMergePlugin(cliapp.Plugin):
                 raise cliapp.AppException(
                     'Command failed at repo %s: %s' % (repo, ' '.join(args)))
 
-    def show_system_branch(self, args):
-        '''Show the name of the current system branch.'''
-
-        branch, dirname = self.deduce_system_branch()
-        self.app.output.write('%s\n' % branch)
-
-    def show_branch_root(self, args):
-        '''Show the name of the repository holding the system morphologies.
-
-        This would, for example, write out something like:
-
-            /src/ws/master/baserock:baserock/morphs
-
-        when the master branch of the `baserock:baserock/morphs`
-        repository is checked out.
-
-        '''
-
-        workspace = self.deduce_workspace()
-        system_branch, branch_dir = self.deduce_system_branch()
-        branch_root = self.get_branch_config(branch_dir, 'branch.root')
-        self.app.output.write('%s\n' % branch_root)
diff --git a/morphlib/sysbranchdir.py b/morphlib/sysbranchdir.py
new file mode 100644
index 00000000..e4af53cf
--- /dev/null
+++ b/morphlib/sysbranchdir.py
@@ -0,0 +1,219 @@
+# Copyright (C) 2013  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# =*= License: GPL-2 =*=
+
+
+import cliapp
+import os
+import urlparse
+import uuid
+
+import morphlib
+
+
+class SystemBranchDirectoryAlreadyExists(morphlib.Error):
+
+    def __init__(self, root_directory):
+        self.msg = (
+            "%s: File exists" % 
+            root_directory)
+
+
+class NotInSystemBranch(morphlib.Error):
+
+    def __init__(self, dirname):
+        self.msg = (
+            "Can't find the system branch directory.\n"
+            "Morph must be built and deployed within "
+            "the system branch checkout.")
+
+
+class SystemBranchDirectory(object):
+
+    '''A directory containing a checked out system branch.'''
+
+    def __init__(self,
+        root_directory, root_repository_url, system_branch_name):
+        self.root_directory = root_directory
+        self.root_repository_url = root_repository_url
+        self.system_branch_name = system_branch_name
+
+    @property
+    def _magic_path(self):
+        return os.path.join(self.root_directory, '.morph-system-branch')
+
+    @property
+    def _config_path(self):
+        return os.path.join(self._magic_path, 'config')
+
+    def set_config(self, key, value):
+        '''Set a configuration key/value pair.'''
+        cliapp.runcmd(['git', 'config', '-f', self._config_path, key, value])
+
+    def get_config(self, key):
+        '''Get a configuration value for a given key.'''
+        value = cliapp.runcmd(['git', 'config', '-f', self._config_path, key])
+        return value.strip()
+
+    def get_git_directory_name(self, repo_url):
+        '''Return directory pathname for a given git repository.
+
+        If the URL is a real one (not aliased), the schema and leading //
+        are removed from it, as is a .git suffix.
+
+        '''
+
+        # Parse the URL. If the path component is absolute, we assume
+        # it's a real URL; otherwise, an aliased URL.
+        parts = urlparse.urlparse(repo_url)
+
+        if os.path.isabs(parts.path):
+            # Remove .git suffix, if any.
+            path = parts.path
+            if path.endswith('.git'):
+                path = path[:-len('.git')]
+
+            # Add the domain name etc (netloc). Ignore any other parts.
+            # Note that we _know_ the path starts with a slash, so we avoid
+            # adding one here.
+            relative = '%s%s' % (parts.netloc, path)
+        else:
+            relative = repo_url
+
+        # Remove anyleading slashes, or os.path.join below will only
+        # use the relative part (since it's absolute, not relative).
+        while relative.startswith('/'):
+            relative = relative[1:]
+
+        return os.path.join(self.root_directory, relative)
+
+    def clone_cached_repo(self, cached_repo, git_branch_name, checkout_ref):
+        '''Clone a cached git repository into the system branch directory.
+
+        The cloned repository will NOT have the system branch's git branch
+        checked out: instead, checkout_ref is checked out (this is for
+        backwards compatibility with older implementation of "morph
+        branch"; it may change later). The system branch's git branch
+        is NOT created: the caller will need to do that. Submodules are
+        NOT checked out.
+
+        The "origin" remote will be set to follow the cached repository's
+        upstream. Remotes are not updated.
+
+        '''
+
+        # Do the clone.
+        dirname = self.get_git_directory_name(cached_repo.original_name)
+        gd = morphlib.gitdir.clone_from_cached_repo(
+            cached_repo, dirname, checkout_ref)
+
+        # Remember the repo name we cloned from in order to be able
+        # to identify the repo again later using the same name, even
+        # if the user happens to rename the directory.
+        gd.set_config('morph.repository', cached_repo.original_name)
+
+        # Create a UUID for the clone. We will use this for naming
+        # temporary refs, e.g. for building.
+        gd.set_config('morph.uuid', uuid.uuid4().hex)
+
+        # Configure the "origin" remote to use the upstream git repository,
+        # and not the locally cached copy.
+        resolver = morphlib.repoaliasresolver.RepoAliasResolver(
+            cached_repo.app.settings['repo-alias'])
+        gd.set_remote_fetch_url('origin', resolver.pull_url(cached_repo.url))
+        gd.set_config(
+            'url.%s.pushInsteadOf' %
+                resolver.push_url(cached_repo.original_name),
+            resolver.pull_url(cached_repo.url))
+
+        return gd
+
+    def list_git_directories(self):
+        '''List all git directories in a system branch directory.
+
+        The list will contain zero or more GitDirectory objects.
+
+        '''
+
+        gitdirs = []
+        for dirname, subdirs, filenames in os.walk(self.root_directory):
+            if os.path.isdir(os.path.join(dirname, '.git')):
+                del subdirs[:]
+                gitdirs.append(morphlib.gitdir.GitDirectory(dirname))
+
+        return gitdirs
+
+
+def create(root_directory, root_repository_url, system_branch_name):
+    '''Create a new system branch directory on disk.
+
+    Return a SystemBranchDirectory object that represents the directory.
+
+    The directory MUST NOT exist already. If it does,
+    SystemBranchDirectoryAlreadyExists is raised.
+
+    Note that this does NOT check out the root repository, or do any
+    other git cloning.
+
+    '''
+
+    if os.path.exists(root_directory):
+        raise SystemBranchDirectoryAlreadyExists(root_directory)
+
+    magic_dir = os.path.join(root_directory, '.morph-system-branch')
+    os.makedirs(root_directory)
+    os.mkdir(magic_dir)
+
+    sb = SystemBranchDirectory(
+        root_directory, root_repository_url, system_branch_name)
+    sb.set_config('branch.name', system_branch_name)
+    sb.set_config('branch.root', root_repository_url)
+    sb.set_config('branch.uuid', uuid.uuid4().hex)
+
+    return sb
+
+
+def open(root_directory):
+    '''Open an existing system branch directory.'''
+
+    # Ugly hack follows.
+    sb = SystemBranchDirectory(root_directory, None, None)
+    root_repository_url = sb.get_config('branch.root')
+    system_branch_name = sb.get_config('branch.name')
+
+    return SystemBranchDirectory(
+        root_directory, root_repository_url, system_branch_name)
+
+
+def open_from_within(dirname):
+    '''Open a system branch directory, given any directory.
+
+    The directory can be within the system branch root directory,
+    or it can be a parent, in some cases. If each parent on the
+    path from dirname to the system branch root directory has no
+    siblings, this function will find it.
+
+    '''
+
+    root_directory = morphlib.util.find_root(
+        dirname, '.morph-system-branch')
+    if root_directory is None:
+        root_directory = morphlib.util.find_leaf(
+            dirname, '.morph-system-branch')
+    if root_directory is None:
+        raise NotInSystemBranch(dirname)
+    return morphlib.sysbranchdir.open(root_directory)
+
diff --git a/morphlib/sysbranchdir_tests.py b/morphlib/sysbranchdir_tests.py
new file mode 100644
index 00000000..8e62791f
--- /dev/null
+++ b/morphlib/sysbranchdir_tests.py
@@ -0,0 +1,213 @@
+# Copyright (C) 2013  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# =*= License: GPL-2 =*=
+
+
+import cliapp
+import os
+import shutil
+import tempfile
+import unittest
+
+import morphlib
+
+
+class SystemBranchDirectoryTests(unittest.TestCase):
+
+    def setUp(self):
+        self.tempdir = tempfile.mkdtemp()
+        self.root_directory = os.path.join(self.tempdir, 'rootdir')
+        self.root_repository_url = 'test:morphs'
+        self.system_branch_name = 'foo/bar'
+
+    def tearDown(self):
+        shutil.rmtree(self.tempdir)
+
+    def create_fake_cached_repo(self):
+
+        class FakeCachedRepo(object):
+
+            def __init__(self, url, path):
+                self.app = self
+                self.settings = {
+                    'repo-alias': [],
+                }
+                self.original_name = url
+                self.url = 'git://blahlbah/blah/blahblahblah.git'
+                self.path = path
+
+                os.mkdir(self.path)
+                cliapp.runcmd(['git', 'init', self.path])
+                with open(os.path.join(self.path, 'filename'), 'w') as f:
+                    f.write('this is a file\n')
+                cliapp.runcmd(['git', 'add', 'filename'], cwd=self.path)
+                cliapp.runcmd(
+                    ['git', 'commit', '-m', 'initial'], cwd=self.path)
+
+            def clone_checkout(self, ref, target_dir):
+                cliapp.runcmd(
+                    ['git', 'clone', '-b', ref, self.path, target_dir])
+
+        subdir = tempfile.mkdtemp(dir=self.tempdir)
+        path = os.path.join(subdir, 'foo')
+        return FakeCachedRepo(self.root_repository_url, path)
+
+    def test_creates_system_branch_directory(self):
+        sb = morphlib.sysbranchdir.create(
+            self.root_directory,
+            self.root_repository_url,
+            self.system_branch_name)
+        self.assertEqual(sb.root_directory, self.root_directory)
+        self.assertEqual(sb.root_repository_url, self.root_repository_url)
+        self.assertEqual(sb.system_branch_name, self.system_branch_name)
+
+        magic_dir = os.path.join(self.root_directory, '.morph-system-branch')
+        self.assertTrue(os.path.isdir(self.root_directory))
+        self.assertTrue(os.path.isdir(magic_dir))
+        self.assertTrue(os.path.isfile(os.path.join(magic_dir, 'config')))
+        self.assertEqual(
+            sb.get_config('branch.root'), self.root_repository_url)
+        self.assertEqual(
+            sb.get_config('branch.name'), self.system_branch_name)
+        self.assertTrue(sb.get_config('branch.uuid'))
+
+    def test_opens_system_branch_directory(self):
+        morphlib.sysbranchdir.create(
+            self.root_directory,
+            self.root_repository_url,
+            self.system_branch_name)
+        sb = morphlib.sysbranchdir.open(self.root_directory)
+        self.assertEqual(sb.root_directory, self.root_directory)
+        self.assertEqual(sb.root_repository_url, self.root_repository_url)
+        self.assertEqual(sb.system_branch_name, self.system_branch_name)
+
+    def test_opens_system_branch_directory_from_a_subdirectory(self):
+        morphlib.sysbranchdir.create(
+            self.root_directory,
+            self.root_repository_url,
+            self.system_branch_name)
+        subdir = os.path.join(self.root_directory, 'a', 'b', 'c')
+        os.makedirs(subdir)
+        sb = morphlib.sysbranchdir.open_from_within(subdir)
+        self.assertEqual(sb.root_directory, self.root_directory)
+        self.assertEqual(sb.root_repository_url, self.root_repository_url)
+        self.assertEqual(sb.system_branch_name, self.system_branch_name)
+
+    def test_fails_opening_system_branch_directory_when_none_exists(self):
+        self.assertRaises(
+            morphlib.sysbranchdir.NotInSystemBranch,
+            morphlib.sysbranchdir.open_from_within,
+            self.tempdir)
+
+    def test_opens_system_branch_directory_when_it_is_the_only_child(self):
+        deep_root = os.path.join(self.tempdir, 'a', 'b', 'c')
+        morphlib.sysbranchdir.create(
+            deep_root,
+            self.root_repository_url,
+            self.system_branch_name)
+        sb = morphlib.sysbranchdir.open(deep_root)
+        self.assertEqual(sb.root_directory, deep_root)
+        self.assertEqual(sb.root_repository_url, self.root_repository_url)
+        self.assertEqual(sb.system_branch_name, self.system_branch_name)
+
+    def test_fails_to_create_if_directory_already_exists(self):
+        os.mkdir(self.root_directory)
+        self.assertRaises(
+            morphlib.sysbranchdir.SystemBranchDirectoryAlreadyExists,
+            morphlib.sysbranchdir.create,
+            self.root_directory,
+            self.root_repository_url,
+            self.system_branch_name)
+
+    def test_sets_and_gets_configuration_values(self):
+        sb = morphlib.sysbranchdir.create(
+            self.root_directory,
+            self.root_repository_url,
+            self.system_branch_name)
+        sb.set_config('foo.key', 'foovalue')
+
+        sb2 = morphlib.sysbranchdir.open(self.root_directory)
+        self.assertEqual(sb2.get_config('foo.key'), 'foovalue')
+
+    def test_reports_correct_name_for_git_directory_from_aliases_url(self):
+        sb = morphlib.sysbranchdir.create(
+            self.root_directory,
+            self.root_repository_url,
+            self.system_branch_name)
+        self.assertEqual(
+            sb.get_git_directory_name('baserock:baserock/morph'),
+            os.path.join(self.root_directory, 'baserock:baserock/morph'))
+
+    def test_reports_correct_name_for_git_directory_from_real_url(self):
+        stripped = 'git.baserock.org/baserock/baserock/morph'
+        url = 'git://%s.git' % stripped
+        sb = morphlib.sysbranchdir.create(
+            self.root_directory,
+            url,
+            self.system_branch_name)
+        self.assertEqual(
+            sb.get_git_directory_name(url),
+            os.path.join(self.root_directory, stripped))
+
+    def test_reports_correct_name_for_git_directory_from_file_url(self):
+        stripped = 'foobar/morphs'
+        url = 'file:///%s.git' % stripped
+        sb = morphlib.sysbranchdir.create(
+            self.root_directory,
+            url,
+            self.system_branch_name)
+        self.assertEqual(
+            sb.get_git_directory_name(url),
+            os.path.join(self.root_directory, stripped))
+
+    def test_clones_git_repository(self):
+
+        sb = morphlib.sysbranchdir.create(
+            self.root_directory,
+            self.root_repository_url,
+            self.system_branch_name)
+
+        cached_repo = self.create_fake_cached_repo()
+        gd = sb.clone_cached_repo(
+            cached_repo, self.system_branch_name, 'master')
+
+        self.assertEqual(
+            gd.dirname,
+            sb.get_git_directory_name(cached_repo.original_name))
+
+    def test_lists_git_directories(self):
+
+        def fake_git_clone(dirname, url, branch):
+            os.mkdir(dirname)
+            subdir = os.path.join(dirname, '.git')
+            os.mkdir(subdir)
+
+        sb = morphlib.sysbranchdir.create(
+            self.root_directory,
+            self.root_repository_url,
+            self.system_branch_name)
+
+        sb._git_clone = fake_git_clone
+
+        cached_repo = self.create_fake_cached_repo()
+        sb.clone_cached_repo(cached_repo, 'branch1', 'master')
+
+        gd_list = sb.list_git_directories()
+        self.assertEqual(len(gd_list), 1)
+        self.assertEqual(
+            gd_list[0].dirname,
+            sb.get_git_directory_name(cached_repo.original_name))
+
diff --git a/morphlib/util.py b/morphlib/util.py
index a9c22217..ead0bafe 100644
--- a/morphlib/util.py
+++ b/morphlib/util.py
@@ -197,28 +197,28 @@ def copyfileobj(inputfp, outputfp, blocksize=1024*1024):  # pragma: no cover
     while 1:
         inbuf = inputfp.read(blocksize)
         if not inbuf: break
-        if not buf: 
+        if not buf:
             buf = inbuf
         else:
             buf += inbuf
-            
+
         # Combine "short" reads
         if (len(buf) < blocksize):
             continue
-            
+
         buflen = len(buf)
         if buf == "\x00" * buflen:
             outputfp.seek(buflen, os.SEEK_CUR)
             buf = None
             # flag sparse=True, that we seek()ed, but have not written yet
             # The filesize is wrong until we write
-            sparse = True 
+            sparse = True
         else:
             outputfp.write(buf)
             buf = None
             # We wrote, so clear sparse.
             sparse = False
-            
+
     if buf:
         outputfp.write(buf)
     elif sparse:
@@ -244,7 +244,7 @@ def on_same_filesystem(path_a, path_b): # pragma: no cover
 def unify_space_requirements(tmp_path, tmp_min_size,
                              cache_path, cache_min_size): # pragma: no cover
     """Adjust minimum sizes when paths share a disk.
-    
+
        Given pairs of path and minimum size, return the minimum sizes such
        that when the paths are on the same disk, the sizes are added together.
 
@@ -280,3 +280,50 @@ def check_disk_available(tmp_path, tmp_min_size,
                          'space or reduce the disk space required by the '
                          'tempdir-min-space and cachedir-min-space '
                          'configuration options.')
+
+
+
+
+def find_root(dirname, subdir_name):
+    '''Find parent of a directory, at or above a given directory.
+
+    The sought-after directory is indicated by the existence of a
+    subdirectory of the indicated name. For example, dirname might
+    be the current working directory of the process, and subdir_name
+    might be ".morph"; then the returned value would be the Morph
+    workspace root directory, which has a subdirectory called
+    ".morph".
+
+    Return path to desired directory, or None if not found.
+
+    '''
+
+    dirname = os.path.normpath(os.path.abspath(dirname))
+    while not os.path.isdir(os.path.join(dirname, subdir_name)):
+        if dirname == '/':
+            return None
+        dirname = os.path.dirname(dirname)
+    return dirname
+
+
+def find_leaf(dirname, subdir_name):
+    '''This is like find_root, except it looks towards leaves.
+
+    It only looks in a subdirectory if it is the only subdirectory.
+    If there are no subdirectories, or more than one, fail.
+    (Subdirectories whose name starts with a dot are ignored for this.)
+
+    '''
+
+    while True:
+        if os.path.exists(os.path.join(dirname, subdir_name)):
+            return dirname
+        pathnames = [
+            os.path.join(dirname, x)
+            for x in os.listdir(dirname)
+            if not x.startswith('.')]
+        subdirs = [x for x in pathnames if os.path.isdir(x)]
+        if len(subdirs) != 1:
+            return None
+        dirname = subdirs[0]
+
diff --git a/morphlib/util_tests.py b/morphlib/util_tests.py
index 89fe184e..eaff0821 100644
--- a/morphlib/util_tests.py
+++ b/morphlib/util_tests.py
@@ -14,6 +14,9 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 
+import os
+import shutil
+import tempfile
 import unittest
 
 import morphlib
@@ -48,3 +51,39 @@ class MakeConcurrencyTests(unittest.TestCase):
 
     def test_returns_6_for_4_cores(self):
         self.assertEqual(morphlib.util.make_concurrency(cores=4), 6)
+
+
+class FindParentOfTests(unittest.TestCase):
+
+    def setUp(self):
+        self.tempdir = tempfile.mkdtemp()
+        os.makedirs(os.path.join(self.tempdir, 'a', 'b', 'c'))
+        self.a = os.path.join(self.tempdir, 'a')
+        self.b = os.path.join(self.tempdir, 'a', 'b')
+        self.c = os.path.join(self.tempdir, 'a', 'b', 'c')
+
+    def tearDown(self):
+        shutil.rmtree(self.tempdir)
+
+    def test_find_root_finds_starting_directory(self):
+        os.mkdir(os.path.join(self.a, '.magic'))
+        self.assertEqual(morphlib.util.find_root(self.a, '.magic'), self.a)
+
+    def test_find_root_finds_ancestor(self):
+        os.mkdir(os.path.join(self.a, '.magic'))
+        self.assertEqual(morphlib.util.find_root(self.c, '.magic'), self.a)
+
+    def test_find_root_returns_none_if_not_found(self):
+        self.assertEqual(morphlib.util.find_root(self.c, '.magic'), None)
+
+    def test_find_leaf_finds_starting_directory(self):
+        os.mkdir(os.path.join(self.a, '.magic'))
+        self.assertEqual(morphlib.util.find_leaf(self.a, '.magic'), self.a)
+
+    def test_find_leaf_finds_child(self):
+        os.mkdir(os.path.join(self.c, '.magic'))
+        self.assertEqual(morphlib.util.find_leaf(self.a, '.magic'), self.c)
+
+    def test_find_leaf_returns_none_if_not_found(self):
+        self.assertEqual(morphlib.util.find_leaf(self.a, '.magic'), None)
+
diff --git a/morphlib/workspace.py b/morphlib/workspace.py
index 3a2269c8..93f699e6 100644
--- a/morphlib/workspace.py
+++ b/morphlib/workspace.py
@@ -53,6 +53,40 @@ class Workspace(object):
     def __init__(self, root_directory):
         self.root = root_directory
 
+    def get_default_system_branch_directory_name(self, system_branch_name):
+        '''Determine directory where a system branch would be checked out.
+
+        Return the fully qualified pathname to the directory where
+        a system branch would be checked out. The directory may or may
+        not exist already.
+
+        If the system branch is checked out, but into a directory of
+        a different name (which is allowed), that is ignored: this method
+        only computed the default name.
+
+        '''
+
+        return os.path.join(self.root, system_branch_name)
+
+    def create_system_branch_directory(self,
+        root_repository_url, system_branch_name):
+        '''Create a directory for a system branch.
+
+        Return a SystemBranchDirectory object that represents the
+        directory. The directory must not already exist. The directory
+        gets created and initialised (the .morph-system-branch/config
+        file gets created and populated). The root repository of the
+        system branch does NOT get checked out, the caller needs to
+        do that.
+
+        '''
+
+        dirname = self.get_default_system_branch_directory_name(
+            system_branch_name)
+        sb = morphlib.sysbranchdir.create(
+            dirname, root_repository_url, system_branch_name)
+        return sb
+
 
 def open(dirname):
     '''Open an existing workspace.
diff --git a/morphlib/workspace_tests.py b/morphlib/workspace_tests.py
index 7837481a..83b5e54f 100644
--- a/morphlib/workspace_tests.py
+++ b/morphlib/workspace_tests.py
@@ -83,3 +83,19 @@ class WorkspaceTests(unittest.TestCase):
             morphlib.workspace.NotInWorkspace,
             morphlib.workspace.open, self.tempdir)
 
+    def test_invents_appropriate_name_for_system_branch_directory(self):
+        self.create_it()
+        ws = morphlib.workspace.open(self.workspace_dir)
+        branch = 'foo/bar'
+        self.assertEqual(
+            ws.get_default_system_branch_directory_name(branch),
+            os.path.join(self.workspace_dir, branch))
+
+    def test_creates_system_branch_directory(self):
+        self.create_it()
+        ws = morphlib.workspace.open(self.workspace_dir)
+        url = 'test:morphs'
+        branch = 'my/new/thing'
+        sb = ws.create_system_branch_directory(url, branch)
+        self.assertTrue(type(sb), morphlib.sysbranchdir.SystemBranchDirectory)
+