Merge branch 'sam/fewer-submodule-updates'

Reviewed-By: Javier Jardón <javier.jardon@codethink.co.uk>
Reviewed-By: Richard Maw <richard.maw@codethink.co.uk>

3 files changed, 65 insertions, 66 deletions

diff --git a/morphlib/app.py b/morphlib/app.py
index 0c87f814..b8bae850 100644
--- a/morphlib/app.py
+++ b/morphlib/app.py
@@ -297,26 +297,6 @@ class Morph(cliapp.Application):
                    morphlib.util.sanitise_morphology_path(args[2]))
             args = args[3:]
 
-    def cache_repo_and_submodules(self, cache, url, ref, done):
-        subs_to_process = set()
-        subs_to_process.add((url, ref))
-        while subs_to_process:
-            url, ref = subs_to_process.pop()
-            done.add((url, ref))
-            cached_repo = cache.cache_repo(url)
-            cached_repo.update()
-
-            try:
-                submodules = morphlib.git.Submodules(self, cached_repo.path,
-                                                     ref)
-                submodules.load()
-            except morphlib.git.NoModulesFileError:
-                pass
-            else:
-                for submod in submodules:
-                    if (submod.url, submod.commit) not in done:
-                        subs_to_process.add((submod.url, submod.commit))
-
     def _write_status(self, text):
         timestamp = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime())
         self.output.write('%s %s\n' % (timestamp, text))
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index 0aa50a3b..8572450d 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -387,39 +387,8 @@ class BuildCommand(object):
         '''Update the local git repository cache with the sources.'''
 
         repo_name = source.repo_name
-        if self.app.settings['no-git-update']:
-            self.app.status(msg='Not updating existing git repository '
-                                '%(repo_name)s '
-                                'because of no-git-update being set',
-                            chatty=True,
-                            repo_name=repo_name)
-            source.repo = self.lrc.get_repo(repo_name)
-            return
-
-        if self.lrc.has_repo(repo_name):
-            source.repo = self.lrc.get_repo(repo_name)
-            try:
-                sha1 = source.sha1
-                source.repo.resolve_ref_to_commit(sha1)
-                self.app.status(msg='Not updating git repository '
-                                    '%(repo_name)s because it '
-                                    'already contains sha1 %(sha1)s',
-                                chatty=True, repo_name=repo_name,
-                                sha1=sha1)
-            except morphlib.gitdir.InvalidRefError:
-                self.app.status(msg='Updating %(repo_name)s',
-                                repo_name=repo_name)
-                source.repo.update()
-        else:
-            self.app.status(msg='Cloning %(repo_name)s',
-                            repo_name=repo_name)
-            source.repo = self.lrc.cache_repo(repo_name)
-
-        # Update submodules.
-        done = set()
-        self.app.cache_repo_and_submodules(
-            self.lrc, source.repo.url,
-            source.sha1, done)
+        source.repo = self.lrc.get_updated_repo(repo_name, ref=source.sha1)
+        self.lrc.ensure_submodules(source.repo, source.sha1)
 
     def cache_artifacts_locally(self, artifacts):
         '''Get artifacts missing from local cache from remote cache.'''
diff --git a/morphlib/localrepocache.py b/morphlib/localrepocache.py
index 39fbd200..1565b913 100644
--- a/morphlib/localrepocache.py
+++ b/morphlib/localrepocache.py
@@ -14,10 +14,7 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 
-import logging
 import os
-import re
-import urllib2
 import urlparse
 import string
 import sys
@@ -246,15 +243,68 @@ class LocalRepoCache(object):
                 return repo
         raise NotCached(reponame)
 
-    def get_updated_repo(self, reponame): # pragma: no cover
-        '''Return object representing cached repository, which is updated.'''
+    def get_updated_repo(self, repo_name, ref=None):  # pragma: no cover
+        '''Return object representing cached repository.
 
-        if not self._app.settings['no-git-update']:
-            cached_repo = self.cache_repo(reponame)
-            self._app.status(
-                msg='Updating git repository %s in cache' % reponame)
-            cached_repo.update()
-        else:
-            cached_repo = self.get_repo(reponame)
-        return cached_repo
+        If 'ref' is None, the repo will be updated unless
+        app.settings['no-git-update'] is set.
+
+        If 'ref' is set to a SHA1, the repo will only be updated if 'ref' isn't
+        already available locally.
 
+        '''
+
+        if self._app.settings['no-git-update']:
+            self._app.status(msg='Not updating existing git repository '
+                                 '%(repo_name)s '
+                                 'because of no-git-update being set',
+                             chatty=True,
+                             repo_name=repo_name)
+            return self.get_repo(repo_name)
+
+        if self.has_repo(repo_name):
+            repo = self.get_repo(repo_name)
+            if ref and morphlib.git.is_valid_sha1(ref):
+                try:
+                    repo.resolve_ref_to_commit(ref)
+                    self._app.status(msg='Not updating git repository '
+                                         '%(repo_name)s because it '
+                                         'already contains sha1 %(sha1)s',
+                                     chatty=True, repo_name=repo_name,
+                                     sha1=ref)
+                    return repo
+                except morphlib.gitdir.InvalidRefError:
+                    pass
+
+            self._app.status(msg='Updating %(repo_name)s',
+                             repo_name=repo_name)
+            repo.update()
+            return repo
+        else:
+            self._app.status(msg='Cloning %(repo_name)s',
+                             repo_name=repo_name)
+            return self.cache_repo(repo_name)
+
+    def ensure_submodules(self, toplevel_repo,
+                          toplevel_ref):  # pragma: no cover
+        '''Ensure any submodules of a given repo are cached and up to date.'''
+
+        def submodules_for_repo(repo_path, ref):
+            try:
+                submodules = morphlib.git.Submodules(self._app, repo_path, ref)
+                submodules.load()
+                return [(submod.url, submod.commit) for submod in submodules]
+            except morphlib.git.NoModulesFileError:
+                return []
+
+        done = set()
+        subs_to_process = submodules_for_repo(toplevel_repo.path, toplevel_ref)
+        while subs_to_process:
+            url, ref = subs_to_process.pop()
+            done.add((url, ref))
+
+            cached_repo = self.get_updated_repo(url, ref=ref)
+
+            for submod in submodules_for_repo(cached_repo.path, ref):
+                if (submod.url, submod.commit) not in done:
+                    subs_to_process.add((submod.url, submod.commit))