Cache submodule repositories recursively.

This commit introduces the following new classes:

morphlib.git.Submodules:
  * takes a parent repo treeish
  * parses the .gitmodule file into morphlib.git.Submodule objects
  * provides iterator/container functionality for submodules

morphlib.git.Submodule:
  * represents a single entry in a .gitmodules file
  * stores a Treeish for the corresponding repository

In addition to this, the exception classes InvalidTreeish and
SourceNotFound where renamed to InvalidReferenceError and
RepositoryUpdateError. Several new exception classes were added for when
resolving submodules fails.

The SourceManager now resolves the Submodules and Submodule objects for
the submodules of a Treeish in SourceManager.get_treeish() and also
takes care of caching submodule repositories whenever necessary.

3 files changed, 190 insertions, 38 deletions

diff --git a/morphlib/git.py b/morphlib/git.py
index 9a03dbed..e64fecb9 100644
--- a/morphlib/git.py
+++ b/morphlib/git.py
@@ -14,11 +14,15 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 
-import logging
 import binascii
-import morphlib
-import os
 import cliapp
+import ConfigParser
+import logging
+import os
+import re
+import StringIO
+
+import morphlib
 
 
 class NoMorphs(Exception):
@@ -35,7 +39,7 @@ class TooManyMorphs(Exception):
                            (repo, ref, ', '.join(morphs)))
 
 
-class InvalidTreeish(cliapp.AppException):
+class InvalidReferenceError(cliapp.AppException):
 
     def __init__(self, repo, ref):
         Exception.__init__(self, '%s is an invalid reference for repo %s' %
@@ -77,7 +81,7 @@ class Treeish(object):
 
     def _is_sha(self, ref):
         if len(ref) != 40:
-            raise InvalidTreeish(self.original_repo, ref)
+            raise InvalidReferenceError(self.original_repo, ref)
 
         try:
                 binascii.unhexlify(ref)
@@ -85,7 +89,118 @@ class Treeish(object):
                 ex.runv(['git', 'rev-list', '--no-walk', ref])
                 self.sha1=ref
         except (TypeError, morphlib.execute.CommandFailure):
-            raise InvalidTreeish(self.original_repo, ref)
+            raise InvalidReferenceError(self.original_repo, ref)
+
+
+class NoModulesFileError(cliapp.AppException):
+
+    def __init__(self, treeish):
+        Exception.__init__(self, '%s has no .gitmodules file.' % treeish)
+
+
+class Submodule(object):
+
+    def __init__(self, parent_treeish, name, url, path):
+        self.parent_treeish = parent_treeish
+        self.name = name
+        self.url = url
+        self.path = path
+
+
+class ModulesFileParseError(cliapp.AppException):
+
+    def __init__(self, treeish, message):
+        Exception.__init__(self, 'Failed to parse %s:.gitmodules: %s' %
+                           (treeish, message))
+
+
+class InvalidSectionError(cliapp.AppException):
+
+    def __init__(self, treeish, section):
+        Exception.__init__(self,
+                           '%s:.gitmodules: Found a misformatted section '
+                           'title: [%s]' % (treeish, section))
+
+
+class MissingSubmoduleCommitError(cliapp.AppException):
+
+    def __init__(self, treeish, submodule):
+        Exception.__init__(self,
+                           '%s:.gitmodules: No commit object found for '
+                           'submodule "%s"' % (treeish, submodule))
+
+
+class Submodules(object):
+
+    def __init__(self, treeish, msg=logging.debug):
+        self.treeish = treeish
+        self.msg = msg
+        self.submodules = []
+
+    def load(self):
+        content = self._read_gitmodules_file()
+
+        io = StringIO.StringIO(content)
+        parser = ConfigParser.RawConfigParser()
+        parser.readfp(io)
+
+        self._validate_and_read_entries(parser)
+        self._resolve_commits()
+
+    def _read_gitmodules_file(self):
+        try:
+            # try to read the .gitmodules file from the repo/ref
+            ex = morphlib.execute.Execute(self.treeish.repo, self.msg)
+            content = ex.runv(['git', 'cat-file', 'blob', '%s:.gitmodules' %
+                               self.treeish.ref])
+
+            # drop indentation in sections, as RawConfigParser cannot handle it
+            return '\n'.join([line.strip() for line in content.splitlines()])
+        except morphlib.execute.CommandFailure:
+            raise NoModulesFileError(self.treeish)
+
+    def _validate_and_read_entries(self, parser):
+        for section in parser.sections():
+            # validate section name against the 'section "foo"' pattern
+            section_pattern = r'submodule "(.*)"'
+            if re.match(section_pattern, section):
+                # parse the submodule name, URL and path
+                name = re.sub(section_pattern, r'\1', section)
+                url = parser.get(section, 'url')
+                path = parser.get(section, 'path')
+
+                # add a submodule object to the list
+                submodule = Submodule(self.treeish, name, url, path)
+                self.submodules.append(submodule)
+            else:
+                raise InvalidSectionError(self.treeish, section)
+
+    def _resolve_commits(self):
+        ex = morphlib.execute.Execute(self.treeish.repo, self.msg)
+        for submodule in self.submodules:
+            try:
+                # list objects in the parent repo tree to find the commit
+                # object that corresponds to the submodule
+                commit = ex.runv(['git', 'ls-tree', self.treeish.ref,
+                                  submodule.name])
+
+                # read the commit hash from the output
+                submodule.commit = commit.split()[2]
+
+                # fail if the commit hash is invalid
+                if len(submodule.commit) != 40:
+                    raise MissingSubmoduleCommitError(self.treeish,
+                                                      submodule.name)
+            except morphlib.execute.CommandFailure:
+                raise MissingSubmoduleCommitError(self.treeish, submodule.name)
+
+    def __iter__(self):
+        for submodule in self.submodules:
+            yield submodule
+
+    def __len__(self):
+        return len(self.submodules)
+
 
 def export_sources(treeish, tar_filename, msg=logging.debug):
     '''Export the contents of a specific commit into a compressed tarball.'''
diff --git a/morphlib/sourcemanager.py b/morphlib/sourcemanager.py
index e780cecb..2f0a4c08 100644
--- a/morphlib/sourcemanager.py
+++ b/morphlib/sourcemanager.py
@@ -37,7 +37,7 @@ def quote_url(url):
     return ''.join([transl(x) for x in url])
 
 
-class SourceNotFound(Exception):
+class RepositoryUpdateError(Exception):
 
     def __init__(self, repo, ref):
         Exception.__init__(self, 'No source found at %s:%s' % (repo, ref))
@@ -123,47 +123,74 @@ class SourceManager(object):
         ex = morphlib.execute.Execute(self.cache_dir, msg=self.msg)
         ex.runv(['wget', '-c', url])
 
+    def _cache_git_from_base_urls(self, repo, ref):
+        treeish = None
+
+        # try all base URLs to load the treeish
+        for base_url in self.settings['git-base-url']:
+            # generate the full repo URL
+            if not base_url.endswith('/'):
+                base_url += '/'
+            full_repo = urlparse.urljoin(base_url, repo)
+
+            self.msg('Updating repository %s' % quote_url(full_repo))
+            self.indent_more()
+
+            # try to clone/update the repo so that we can obtain a treeish
+            success, gitcache = self._get_git_cache(full_repo)
+            if success:
+                treeish = morphlib.git.Treeish(gitcache, repo, ref, self.msg)
+                self.indent_less()
+                break
+
+            self.indent_less()
+
+        if treeish:
+            return treeish
+        else:
+            raise RepositoryUpdateError(repo, ref)
+
+    def _resolve_submodules(self, treeish):
+        self.indent_more()
+
+        # resolve submodules
+        treeish.submodules = morphlib.git.Submodules(treeish, self.msg)
+        try:
+            # load submodules from .gitmodules
+            treeish.submodules.load()
+
+            # resolve the tree-ishes for all submodules recursively
+            for submodule in treeish.submodules: # pragma: no cover
+                submodule.treeish = self.get_treeish(submodule.url,
+                                                     submodule.commit)
+        except morphlib.git.NoModulesFileError:
+            # this is not really an error, the repository simply
+            # does not specify any git submodules
+            pass
+
+        self.indent_less()
+
     def get_treeish(self, repo, ref):
         '''Returns a Treeish for a URL or repo name with a given reference.
 
         If the source hasn't been cloned yet, this will fetch it, either using
         clone or by fetching a bundle. 
         
-        Raises morphlib.git.InvalidTreeish if the reference cannot be found.
-        Raises morphlib.sourcemanager.SourceNotFound if source cannot be found.
+        Raises morphlib.git.InvalidReferenceError if the reference cannot be
+        found. Raises morphlib.sourcemanager.RepositoryUpdateError if the
+        repository cannot be cloned or updated.
 
         '''
 
-        # load the corresponding treeish on demand
         if (repo, ref) not in self.cached_treeishes:
-            # variable for storing the loaded treeish
-            treeish = None
-
-            # try loading it from all base URLs
-            for base_url in self.settings['git-base-url']:
-                # generate the full repo URL
-                if not base_url.endswith('/'):
-                    base_url += '/'
-                full_repo = urlparse.urljoin(base_url, repo)
-                
-                self.msg('Updating repository %s' % quote_url(full_repo))
-                self.indent_more()
-
-                # try to clone/update the repo so that we can obtain a treeish
-                success, gitcache = self._get_git_cache(full_repo)
-                if success:
-                    treeish = morphlib.git.Treeish(gitcache, repo, ref,
-                                                   self.msg)
-                    self.indent_less()
-                    break
+            # load the corresponding treeish on demand
+            treeish = self._cache_git_from_base_urls(repo, ref)
 
-                self.indent_less()
+            # have a treeish now, cache it to avoid loading it twice
+            self.cached_treeishes[(repo, ref)] = treeish
 
-            # if we have a treeish now, cache it to avoid loading it twice
-            if treeish:
-                self.cached_treeishes[(repo, ref)] = treeish
-            else:
-                raise SourceNotFound(repo, ref)
+            # load tree-ishes for submodules, if necessary
+            self._resolve_submodules(treeish)
 
         # we should now have a cached treeish to use now
         return self.cached_treeishes[(repo, ref)]
diff --git a/morphlib/sourcemanager_tests.py b/morphlib/sourcemanager_tests.py
index 9cd7a173..97ef7ed5 100644
--- a/morphlib/sourcemanager_tests.py
+++ b/morphlib/sourcemanager_tests.py
@@ -96,6 +96,15 @@ class SourceManagerTests(unittest.TestCase):
 
         shutil.rmtree(tempdir)
 
+    def test_get_ref_treeish_for_self_without_submodules(self):
+        tempdir = tempfile.mkdtemp()
+
+        s = morphlib.sourcemanager.SourceManager(DummyApp(), tempdir)
+        t = s.get_treeish(self.temprepo, 'master')
+        self.assertEquals(len(t.submodules), 0)
+
+        shutil.rmtree(tempdir)
+
     def test_get_sha1_treeish_for_self_bundle(self):
         tempdir = tempfile.mkdtemp()
         bundle_server_loc = self.temprepodir
@@ -129,8 +138,9 @@ class SourceManagerTests(unittest.TestCase):
             shutil.copy(path, s.cache_dir)
 
         s._wget = wget
-        self.assertRaises(morphlib.sourcemanager.SourceNotFound, s.get_treeish,
-                          'asdf','e28a23812eadf2fce6583b8819b9c5dbd36b9fb9')
+        self.assertRaises(morphlib.sourcemanager.RepositoryUpdateError,
+                          s.get_treeish, 'asdf',
+                          'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9')
 
         shutil.rmtree(tempdir)