Merge branch 'sam/source-resolver-cleanup'

Reviewed-By: Adam Coldrick <adam.coldrick@codethink.co.uk>
Reviewed-By: Richard Maw <richard.maw@codethink.co.uk>

8 files changed, 217 insertions, 148 deletions

diff --git a/morphlib/__init__.py b/morphlib/__init__.py
index f98c11aa..8319430c 100644
--- a/morphlib/__init__.py
+++ b/morphlib/__init__.py
@@ -79,6 +79,7 @@ import repoaliasresolver
 import savefile
 import source
 import sourcepool
+import sourceresolver
 import stagingarea
 import stopwatch
 import sysbranchdir
diff --git a/morphlib/app.py b/morphlib/app.py
index c3c9c970..3f4171a4 100644
--- a/morphlib/app.py
+++ b/morphlib/app.py
@@ -15,13 +15,11 @@
 
 
 import cliapp
-import collections
 import logging
 import os
 import sys
 import time
 import urlparse
-import warnings
 import extensions
 
 import morphlib
@@ -290,135 +288,6 @@ class Morph(cliapp.Application):
                    morphlib.util.sanitise_morphology_path(args[2]))
             args = args[3:]
 
-    def create_source_pool(self, lrc, rrc, repo, ref, filename,
-                           original_ref=None):
-        pool = morphlib.sourcepool.SourcePool()
-
-        def add_to_pool(reponame, ref, filename, absref, tree, morphology):
-            sources = morphlib.source.make_sources(reponame, ref,
-                                                   filename, absref,
-                                                   tree, morphology)
-            for source in sources:
-                pool.add(source)
-
-        self.traverse_morphs(repo, ref, [filename], lrc, rrc,
-                             update=not self.settings['no-git-update'],
-                             visit=add_to_pool,
-                             definitions_original_ref=original_ref)
-        return pool
-
-    def resolve_ref(self, lrc, rrc, reponame, ref, update=True):
-        '''Resolves commit and tree sha1s of the ref in a repo and returns it.
-
-        If update is True then this has the side-effect of updating
-        or cloning the repository into the local repo cache.
-        '''
-        absref = None
-
-        if lrc.has_repo(reponame):
-            repo = lrc.get_repo(reponame)
-            if update and repo.requires_update_for_ref(ref):
-                self.status(msg='Updating cached git repository %(reponame)s '
-                            'for ref %(ref)s', reponame=reponame, ref=ref)
-                repo.update()
-            # If the user passed --no-git-update, and the ref is a SHA1 not
-            # available locally, this call will raise an exception.
-            absref, tree = repo.resolve_ref(ref)
-        elif rrc is not None:
-            try:
-                absref, tree = rrc.resolve_ref(reponame, ref)
-                if absref is not None:
-                    self.status(msg='Resolved %(reponame)s %(ref)s via remote '
-                                'repo cache',
-                                reponame=reponame,
-                                ref=ref,
-                                chatty=True)
-            except BaseException, e:
-                logging.warning('Caught (and ignored) exception: %s' % str(e))
-        if absref is None:
-            if update:
-                self.status(msg='Caching git repository %(reponame)s',
-                            reponame=reponame)
-                repo = lrc.cache_repo(reponame)
-                repo.update()
-            else:
-                repo = lrc.get_repo(reponame)
-            absref, tree = repo.resolve_ref(ref)
-        return absref, tree
-
-    def traverse_morphs(self, definitions_repo, definitions_ref,
-                        system_filenames, lrc, rrc, update=True,
-                        visit=lambda rn, rf, fn, arf, m: None,
-                        definitions_original_ref=None):
-        morph_factory = morphlib.morphologyfactory.MorphologyFactory(lrc, rrc,
-                                                                     self)
-        definitions_queue = collections.deque(system_filenames)
-        chunk_in_definitions_repo_queue = []
-        chunk_in_source_repo_queue = []
-        resolved_refs = {}
-        resolved_morphologies = {}
-
-        # Resolve the (repo, ref) pair for the definitions repo, cache result.
-        definitions_absref, definitions_tree = self.resolve_ref(
-            lrc, rrc, definitions_repo, definitions_ref, update)
-
-        if definitions_original_ref:
-            definitions_ref = definitions_original_ref
-
-        while definitions_queue:
-            filename = definitions_queue.popleft()
-
-            key = (definitions_repo, definitions_absref, filename)
-            if not key in resolved_morphologies:
-                resolved_morphologies[key] = morph_factory.get_morphology(*key)
-            morphology = resolved_morphologies[key]
-
-            visit(definitions_repo, definitions_ref, filename,
-                  definitions_absref, definitions_tree, morphology)
-            if morphology['kind'] == 'cluster':
-                raise cliapp.AppException(
-                    "Cannot build a morphology of type 'cluster'.")
-            elif morphology['kind'] == 'system':
-                definitions_queue.extend(
-                    morphlib.util.sanitise_morphology_path(s['morph'])
-                    for s in morphology['strata'])
-            elif morphology['kind'] == 'stratum':
-                if morphology['build-depends']:
-                    definitions_queue.extend(
-                        morphlib.util.sanitise_morphology_path(s['morph'])
-                        for s in morphology['build-depends'])
-                for c in morphology['chunks']:
-                    if 'morph' not in c:
-                        path = morphlib.util.sanitise_morphology_path(
-                            c.get('morph', c['name']))
-                        chunk_in_source_repo_queue.append(
-                            (c['repo'], c['ref'], path))
-                        continue
-                    chunk_in_definitions_repo_queue.append(
-                        (c['repo'], c['ref'], c['morph']))
-
-        for repo, ref, filename in chunk_in_definitions_repo_queue:
-            if (repo, ref) not in resolved_refs:
-                resolved_refs[repo, ref] = self.resolve_ref(
-                    lrc, rrc, repo, ref, update)
-            absref, tree = resolved_refs[repo, ref]
-            key = (definitions_repo, definitions_absref, filename)
-            if not key in resolved_morphologies:
-                resolved_morphologies[key] = morph_factory.get_morphology(*key)
-            morphology = resolved_morphologies[key]
-            visit(repo, ref, filename, absref, tree, morphology)
-
-        for repo, ref, filename in chunk_in_source_repo_queue:
-            if (repo, ref) not in resolved_refs:
-                resolved_refs[repo, ref] = self.resolve_ref(
-                    lrc, rrc, repo, ref, update)
-            absref, tree = resolved_refs[repo, ref]
-            key = (repo, absref, filename)
-            if key not in resolved_morphologies:
-                resolved_morphologies[key] = morph_factory.get_morphology(*key)
-            morphology = resolved_morphologies[key]
-            visit(repo, ref, filename, absref, tree, morphology)
-
     def cache_repo_and_submodules(self, cache, url, ref, done):
         subs_to_process = set()
         subs_to_process.add((url, ref))
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index 438badb3..a658fc55 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -94,10 +94,11 @@ class BuildCommand(object):
 
         '''
         self.app.status(msg='Creating source pool', chatty=True)
-        srcpool = self.app.create_source_pool(
+        srcpool = morphlib.sourceresolver.create_source_pool(
             self.lrc, self.rrc, repo_name, ref, filename,
-            original_ref=original_ref)
-
+            original_ref=original_ref,
+            update_repos=not self.app.settings['no-git-update'],
+            status_cb=self.app.status)
         return srcpool
 
     def validate_sources(self, srcpool):
diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py
index b0a0528d..dad7238e 100644
--- a/morphlib/morphologyfactory.py
+++ b/morphlib/morphologyfactory.py
@@ -41,14 +41,13 @@ class MorphologyFactory(object):
 
     '''A way of creating morphologies which will provide a default'''
 
-    def __init__(self, local_repo_cache, remote_repo_cache=None, app=None):
+    def __init__(self, local_repo_cache, remote_repo_cache=None,
+                 status_cb=None):
         self._lrc = local_repo_cache
         self._rrc = remote_repo_cache
-        self._app = app
 
-    def status(self, *args, **kwargs):  # pragma: no cover
-        if self._app is not None:
-            self._app.status(*args, **kwargs)
+        null_status_function = lambda **kwargs: None
+        self.status = status_cb or null_status_function
 
     def get_morphology(self, reponame, sha1, filename):
         morph_name = os.path.splitext(os.path.basename(filename))[0]
diff --git a/morphlib/morphologyfactory_tests.py b/morphlib/morphologyfactory_tests.py
index 52d5f598..41d06480 100644
--- a/morphlib/morphologyfactory_tests.py
+++ b/morphlib/morphologyfactory_tests.py
@@ -144,19 +144,13 @@ class FakeLocalRepoCache(object):
         return self.lr
 
 
-class FakeApp(object):
-
-    def status(self, **kwargs):
-        pass
-
-
 class MorphologyFactoryTests(unittest.TestCase):
 
     def setUp(self):
         self.lr = FakeLocalRepo()
         self.lrc = FakeLocalRepoCache(self.lr)
         self.rrc = FakeRemoteRepoCache()
-        self.mf = MorphologyFactory(self.lrc, self.rrc, app=FakeApp())
+        self.mf = MorphologyFactory(self.lrc, self.rrc)
         self.lmf = MorphologyFactory(self.lrc, None)
 
     def nolocalfile(self, *args):
diff --git a/morphlib/plugins/list_artifacts_plugin.py b/morphlib/plugins/list_artifacts_plugin.py
index 0f14a579..713473ac 100644
--- a/morphlib/plugins/list_artifacts_plugin.py
+++ b/morphlib/plugins/list_artifacts_plugin.py
@@ -84,8 +84,10 @@ class ListArtifactsPlugin(cliapp.Plugin):
 
         self.app.status(
             msg='Creating source pool for %s' % system_filename, chatty=True)
-        source_pool = self.app.create_source_pool(
-            self.lrc, self.rrc, repo, ref, system_filename)
+        source_pool = morphlib.SourceResolver.create_source_pool(
+            self.lrc, self.rrc, repo, ref, system_filename,
+            update_repos = not self.app.settings['no-git-update'],
+            status_cb=self.app.status)
 
         self.app.status(
             msg='Resolving artifacts for %s' % system_filename, chatty=True)
diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py
new file mode 100644
index 00000000..8c9fd8b8
--- /dev/null
+++ b/morphlib/sourceresolver.py
@@ -0,0 +1,199 @@
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import cliapp
+
+import collections
+import logging
+
+import morphlib
+
+
+class SourceResolver(object):
+    '''Provides a way of resolving the set of sources for a given system.
+
+    There are two levels of caching involved in resolving the sources to build.
+
+    The canonical source for each source is specified in the build-command
+    (for strata and systems) or in the stratum morphology (for chunks). It will
+    be either a normal URL, or a keyed URL using a repo-alias like
+    'baserock:baserock/definitions'.
+
+    The 'remote repo cache' is a Baserock Trove system. It functions as a
+    normal Git server, but in addition it runs a service on port 8080 called
+    'morph-cache-server' which can resolve refs, list their contents and read
+    specific files from the repos it holds. This allows the SourceResolver to
+    work out how to build something without cloning the whole repo. (If a local
+    build of that source ends up being necessary then it will get cloned into
+    the local cache later on).
+
+    The second layer of caching is the local repository cache, which mirrors
+    entire repositories in $cachedir/gits. If a repo is not in the remote repo
+    cache then it must be present in the local repo cache.
+
+    '''
+
+    def __init__(self, local_repo_cache, remote_repo_cache, update_repos,
+                 status_cb=None):
+        self.lrc = local_repo_cache
+        self.rrc = remote_repo_cache
+
+        self.update = update_repos
+
+        self.status = status_cb
+
+    def resolve_ref(self, reponame, ref):
+        '''Resolves commit and tree sha1s of the ref in a repo and returns it.
+
+        If update is True then this has the side-effect of updating
+        or cloning the repository into the local repo cache.
+        '''
+        absref = None
+
+        if self.lrc.has_repo(reponame):
+            repo = self.lrc.get_repo(reponame)
+            if self.update and repo.requires_update_for_ref(ref):
+                self.status(msg='Updating cached git repository %(reponame)s '
+                            'for ref %(ref)s', reponame=reponame, ref=ref)
+                repo.update()
+            # If the user passed --no-git-update, and the ref is a SHA1 not
+            # available locally, this call will raise an exception.
+            absref, tree = repo.resolve_ref(ref)
+        elif self.rrc is not None:
+            try:
+                absref, tree = self.rrc.resolve_ref(reponame, ref)
+                if absref is not None:
+                    self.status(msg='Resolved %(reponame)s %(ref)s via remote '
+                                'repo cache',
+                                reponame=reponame,
+                                ref=ref,
+                                chatty=True)
+            except BaseException, e:
+                logging.warning('Caught (and ignored) exception: %s' % str(e))
+        if absref is None:
+            if self.update:
+                self.status(msg='Caching git repository %(reponame)s',
+                            reponame=reponame)
+                repo = self.lrc.cache_repo(reponame)
+                repo.update()
+            else:
+                repo = self.lrc.get_repo(reponame)
+            absref, tree = repo.resolve_ref(ref)
+        return absref, tree
+
+    def traverse_morphs(self, definitions_repo, definitions_ref,
+                        system_filenames,
+                        visit=lambda rn, rf, fn, arf, m: None,
+                        definitions_original_ref=None):
+        morph_factory = morphlib.morphologyfactory.MorphologyFactory(
+            self.lrc, self.rrc, self.status)
+        definitions_queue = collections.deque(system_filenames)
+        chunk_in_definitions_repo_queue = []
+        chunk_in_source_repo_queue = []
+        resolved_refs = {}
+        resolved_morphologies = {}
+
+        # Resolve the (repo, ref) pair for the definitions repo, cache result.
+        definitions_absref, definitions_tree = self.resolve_ref(
+            definitions_repo, definitions_ref)
+
+        if definitions_original_ref:
+            definitions_ref = definitions_original_ref
+
+        while definitions_queue:
+            filename = definitions_queue.popleft()
+
+            key = (definitions_repo, definitions_absref, filename)
+            if not key in resolved_morphologies:
+                resolved_morphologies[key] = morph_factory.get_morphology(*key)
+            morphology = resolved_morphologies[key]
+
+            visit(definitions_repo, definitions_ref, filename,
+                  definitions_absref, definitions_tree, morphology)
+            if morphology['kind'] == 'cluster':
+                raise cliapp.AppException(
+                    "Cannot build a morphology of type 'cluster'.")
+            elif morphology['kind'] == 'system':
+                definitions_queue.extend(
+                    morphlib.util.sanitise_morphology_path(s['morph'])
+                    for s in morphology['strata'])
+            elif morphology['kind'] == 'stratum':
+                if morphology['build-depends']:
+                    definitions_queue.extend(
+                        morphlib.util.sanitise_morphology_path(s['morph'])
+                        for s in morphology['build-depends'])
+                for c in morphology['chunks']:
+                    if 'morph' not in c:
+                        path = morphlib.util.sanitise_morphology_path(
+                            c.get('morph', c['name']))
+                        chunk_in_source_repo_queue.append(
+                            (c['repo'], c['ref'], path))
+                        continue
+                    chunk_in_definitions_repo_queue.append(
+                        (c['repo'], c['ref'], c['morph']))
+
+        for repo, ref, filename in chunk_in_definitions_repo_queue:
+            if (repo, ref) not in resolved_refs:
+                resolved_refs[repo, ref] = self.resolve_ref(repo, ref)
+            absref, tree = resolved_refs[repo, ref]
+            key = (definitions_repo, definitions_absref, filename)
+            if not key in resolved_morphologies:
+                resolved_morphologies[key] = morph_factory.get_morphology(*key)
+            morphology = resolved_morphologies[key]
+            visit(repo, ref, filename, absref, tree, morphology)
+
+        for repo, ref, filename in chunk_in_source_repo_queue:
+            if (repo, ref) not in resolved_refs:
+                resolved_refs[repo, ref] = self.resolve_ref(repo, ref)
+            absref, tree = resolved_refs[repo, ref]
+            key = (repo, absref, filename)
+            if key not in resolved_morphologies:
+                resolved_morphologies[key] = morph_factory.get_morphology(*key)
+            morphology = resolved_morphologies[key]
+            visit(repo, ref, filename, absref, tree, morphology)
+
+
+def create_source_pool(lrc, rrc, repo, ref, filename,
+                       original_ref=None, update_repos=True,
+                       status_cb=None):
+    '''Find all the sources involved in building a given system.
+
+    Given a system morphology, this function will traverse the tree of stratum
+    and chunk morphologies that the system points to and create appropriate
+    Source objects. These are added to a new SourcePool object, which is
+    returned.
+
+    Note that Git submodules are not considered 'sources' in the current
+    implementation, and so they must be handled separately.
+
+    The 'lrc' and 'rrc' parameters specify the local and remote Git repository
+    caches used for resolving the sources.
+
+    '''
+    pool = morphlib.sourcepool.SourcePool()
+
+    def add_to_pool(reponame, ref, filename, absref, tree, morphology):
+        sources = morphlib.source.make_sources(reponame, ref,
+                                               filename, absref,
+                                               tree, morphology)
+        for source in sources:
+            pool.add(source)
+
+    resolver = SourceResolver(lrc, rrc, update_repos, status_cb)
+    resolver.traverse_morphs(repo, ref, [filename],
+                             visit=add_to_pool,
+                             definitions_original_ref=original_ref)
+    return pool
diff --git a/without-test-modules b/without-test-modules
index 55e5291d..530deb4f 100644
--- a/without-test-modules
+++ b/without-test-modules
@@ -52,3 +52,7 @@ distbuild/timer_event_source.py
 distbuild/worker_build_scheduler.py
 # Not unit tested, since it needs a full system branch
 morphlib/buildbranch.py
+
+# Requires rather a lot of fake data in order to be unit tested; better to
+# leave it to the functional tests.
+morphlib/sourceresolver.py