Unify local and remote repo cache modules

There's not really any reason you'd want to use the RemoteRepoCache
class except as a workaround for the slow speed of some LocalRepoCache
operations, so I can't see this ruining anyone's day.

The main reason for doing this is so we can simply the sourceresolver
code. One reason that the sourceresolver class is so hopelessly
complicated is that it right now has to use two incompatible interfaces
for Git repo caches.

I've taken the opportunity to detangle the RepoCache class from the
App class. Now all of the configuration for the RepoCache class is
passed into the constructor explicitly. This makes the class usable from
outside Morph:

    resolver = morphlib.repoaliasresolver.RepoAliasResolver(aliases=[])
    repo_cache = morphlib.repocache.RepoCache('/src/cache/gits', resolver)

Change-Id: I596c81d7645b67504c88e555172a8c238f4f8a66

9 files changed, 50 insertions, 80 deletions

diff --git a/morphlib/plugins/anchor_plugin.py b/morphlib/plugins/anchor_plugin.py
index 7465c479..a9d07b39 100644
--- a/morphlib/plugins/anchor_plugin.py
+++ b/morphlib/plugins/anchor_plugin.py
@@ -137,9 +137,8 @@ class AnchorPlugin(cliapp.Plugin):
             for reponame, sources in sources_by_reponame.iteritems():
                 # UGLY HACK we need to push *FROM* our local repo cache to
                 # avoid cloning everything multiple times.
-                repo = bc.lrc.get_updated_repo(reponame,
-                                               refs=(s.original_ref
-                                                     for s in sources))
+                repo = bc.repo_cache.get_updated_repo(
+                    reponame, refs=(s.original_ref for s in sources))
                 remote = Remote(repo)
 
                 push_url = resolver.push_url(reponame)
diff --git a/morphlib/plugins/artifact_inspection_plugin.py b/morphlib/plugins/artifact_inspection_plugin.py
index 413a0072..d396f93b 100644
--- a/morphlib/plugins/artifact_inspection_plugin.py
+++ b/morphlib/plugins/artifact_inspection_plugin.py
@@ -36,28 +36,23 @@ class NotASystemArtifactError(cliapp.AppException):
 
 class ProjectVersionGuesser(object):
 
-    def __init__(self, app, lrc, rrc, interesting_files):
+    def __init__(self, app, repo_cache, interesting_files):
         self.app = app
-        self.lrc = lrc
-        self.rrc = rrc
+        self.repo_cache = repo_cache
         self.interesting_files = interesting_files
 
     def file_contents(self, repo, ref, tree):
         filenames = [x for x in self.interesting_files if x in tree]
-        if filenames:
-            if self.lrc.has_repo(repo):
-                repository = self.lrc.get_updated_repo(repo, ref)
-                for filename in filenames:
-                    yield filename, repository.read_file(filename, ref)
-            elif self.rrc:
-                for filename in filenames:
-                    yield filename, self.rrc.cat_file(repo, ref, filename)
+        for filename in filenames:
+            # This can use a remote repo cache if available, to avoid having
+            # to clone every repo locally.
+            yield filename, self.repo_cache.cat_file(repo, ref, filename)
 
 
 class AutotoolsVersionGuesser(ProjectVersionGuesser):
 
-    def __init__(self, app, lrc, rrc):
-        ProjectVersionGuesser.__init__(self, app, lrc, rrc, [
+    def __init__(self, app, repo_cache):
+        ProjectVersionGuesser.__init__(self, app, repo_cache, [
             'configure.ac',
             'configure.in',
             'configure.ac.in',
@@ -136,9 +131,9 @@ class VersionGuesser(object):
 
     def __init__(self, app):
         self.app = app
-        self.lrc, self.rrc = morphlib.util.new_repo_caches(app)
+        self.repo_cache = morphlib.util.new_repo_cache(app)
         self.guessers = [
-            AutotoolsVersionGuesser(app, self.lrc, self.rrc)
+            AutotoolsVersionGuesser(app, self.repo_cache)
         ]
 
     def guess_version(self, repo, ref):
@@ -146,14 +141,10 @@ class VersionGuesser(object):
                         repo=repo, ref=ref, chatty=True)
         version = None
         try:
-            if self.lrc.has_repo(repo):
-                repository = self.lrc.get_updated_repo(repo, ref)
-                tree = repository.list_files(ref=ref, recurse=False)
-            elif self.rrc:
-                repository = None
-                tree = self.rrc.ls_tree(repo, ref)
-            else:
-                return None
+            # This can use a remote repo cache if available, to avoid having
+            # to clone every repo locally.
+            tree = self.repo_cache.ls_tree(repo, ref)
+
             for guesser in self.guessers:
                 version = guesser.guess_version(repo, ref, tree)
                 if version:
diff --git a/morphlib/plugins/certify_plugin.py b/morphlib/plugins/certify_plugin.py
index 735d0332..72d24a51 100644
--- a/morphlib/plugins/certify_plugin.py
+++ b/morphlib/plugins/certify_plugin.py
@@ -57,7 +57,7 @@ class CertifyPlugin(cliapp.Plugin):
         system_filenames = map(morphlib.util.sanitise_morphology_path,
                                args[2:])
 
-        self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app)
+        self.repo_cache = morphlib.util.new_repo_cache(self.app)
         self.resolver = morphlib.artifactresolver.ArtifactResolver()
 
         for system_filename in system_filenames:
@@ -69,9 +69,7 @@ class CertifyPlugin(cliapp.Plugin):
         self.app.status(
             msg='Creating source pool for %s' % system_filename, chatty=True)
         source_pool = morphlib.sourceresolver.create_source_pool(
-            self.lrc, self.rrc, repo, ref, [system_filename],
-            cachedir=self.app.settings['cachedir'],
-            update_repos = not self.app.settings['no-git-update'],
+            self.repo_cache, repo, ref, [system_filename],
             status_cb=self.app.status)
 
         self.app.status(
@@ -115,7 +113,7 @@ class CertifyPlugin(cliapp.Plugin):
                               .format(name, ref))
                 certified = False
 
-            cached = self.lrc.get_updated_repo(source.repo_name, ref)
+            cached = self.repo_cache.get_updated_repo(source.repo_name, ref)
 
             # Test that sha1 ref is anchored in a tag or branch,
             # and thus not a candidate for removal on `git gc`.
diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py
index 273e677d..8b8fbb2d 100644
--- a/morphlib/plugins/cross-bootstrap_plugin.py
+++ b/morphlib/plugins/cross-bootstrap_plugin.py
@@ -304,7 +304,7 @@ class CrossBootstrapPlugin(cliapp.Plugin):
             system_artifact.source, build_env, use_chroot=False)
         builder = BootstrapSystemBuilder(
             self.app, staging_area, build_command.lac, build_command.rac,
-            system_artifact.source, build_command.lrc, 1, False)
+            system_artifact.source, build_command.repo_cache, 1, False)
         builder.build_and_cache()
 
         self.app.status(
diff --git a/morphlib/plugins/diff_plugin.py b/morphlib/plugins/diff_plugin.py
index 26964df8..24a6d69a 100644
--- a/morphlib/plugins/diff_plugin.py
+++ b/morphlib/plugins/diff_plugin.py
@@ -22,7 +22,6 @@ from morphlib.cmdline_parse_utils import (definition_lists_synopsis,
 from morphlib.morphologyfinder import MorphologyFinder
 from morphlib.morphloader import MorphologyLoader
 from morphlib.morphset import MorphologySet
-from morphlib.util import new_repo_caches
 
 
 class DiffPlugin(cliapp.Plugin):
@@ -60,9 +59,10 @@ class DiffPlugin(cliapp.Plugin):
                         name, from_source.repo_name, to_source.repo_name))
 
             if from_source.original_ref != to_source.original_ref:
-                from_repo, to_repo = (self.bc.lrc.get_updated_repo(s.repo_name,
-                                                                   ref=s.sha1)
-                                        for s in (from_source, to_source))
+                repo_cache = self.bc.repo_cache
+                from_repo, to_repo = (repo_cache.get_updated_repo(s.repo_name,
+                                                                  ref=s.sha1)
+                                      for s in (from_source, to_source))
 
                 from_desc = from_repo.version_guess(from_source.sha1)
                 to_desc = to_repo.version_guess(to_source.sha1)
@@ -100,7 +100,7 @@ class DiffPlugin(cliapp.Plugin):
         def get_systems((reponame, ref, definitions)):
             'Convert a definition path list into a list of systems'
             ml = MorphologyLoader()
-            repo = self.bc.lrc.get_updated_repo(reponame, ref=ref)
+            repo = self.bc.repo_cache.get_updated_repo(reponame, ref=ref)
             mf = MorphologyFinder(gitdir=repo, ref=ref)
             # We may have been given an empty set of definitions as input, in
             # which case we instead use every we find.
diff --git a/morphlib/plugins/get_repo_plugin.py b/morphlib/plugins/get_repo_plugin.py
index fc81d6e5..ce0b7af0 100644
--- a/morphlib/plugins/get_repo_plugin.py
+++ b/morphlib/plugins/get_repo_plugin.py
@@ -101,9 +101,9 @@ class GetRepoPlugin(cliapp.Plugin):
                         '%(stratum)s stratum',
                     ref=ref or chunk_spec['ref'], chunk=chunk_spec['name'],
                     stratum=morph['name'])
-                lrc, rrc = morphlib.util.new_repo_caches(self.app)
-                cached_repo = lrc.get_updated_repo(chunk_spec['repo'],
-                                                   chunk_spec['ref'])
+                repo_cache = morphlib.util.new_repo_cache(self.app)
+                cached_repo = repo_cache.get_updated_repo(chunk_spec['repo'],
+                                                          chunk_spec['ref'])
 
                 try:
                     self._clone_repo(cached_repo, dirname,
diff --git a/morphlib/plugins/list_artifacts_plugin.py b/morphlib/plugins/list_artifacts_plugin.py
index c2e6b459..2c098c2a 100644
--- a/morphlib/plugins/list_artifacts_plugin.py
+++ b/morphlib/plugins/list_artifacts_plugin.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2015  Codethink Limited
+# Copyright (C) 2014-2016  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -58,7 +58,7 @@ class ListArtifactsPlugin(cliapp.Plugin):
         system_filenames = map(morphlib.util.sanitise_morphology_path,
                                args[2:])
 
-        self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app)
+        self.repo_cache = morphlib.util.new_repo_cache(self.app)
         self.resolver = morphlib.artifactresolver.ArtifactResolver()
 
         artifact_files = set()
@@ -85,9 +85,7 @@ class ListArtifactsPlugin(cliapp.Plugin):
         self.app.status(
             msg='Creating source pool for %s' % system_filename, chatty=True)
         source_pool = morphlib.sourceresolver.create_source_pool(
-            self.lrc, self.rrc, repo, ref, [system_filename],
-            cachedir=self.app.settings['cachedir'],
-            update_repos = not self.app.settings['no-git-update'],
+            self.repo_cache, repo, ref, [system_filename],
             status_cb=self.app.status)
 
         self.app.status(
diff --git a/morphlib/plugins/show_dependencies_plugin.py b/morphlib/plugins/show_dependencies_plugin.py
index 42f8f273..bfe4d6c2 100644
--- a/morphlib/plugins/show_dependencies_plugin.py
+++ b/morphlib/plugins/show_dependencies_plugin.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2015  Codethink Limited
+# Copyright (C) 2012-2016  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -42,24 +42,6 @@ class ShowDependenciesPlugin(cliapp.Plugin):
         of build dependencies of the constituent components.
 
         '''
-
-        if not os.path.exists(self.app.settings['cachedir']):
-            os.mkdir(self.app.settings['cachedir'])
-        cachedir = os.path.join(self.app.settings['cachedir'], 'gits')
-        tarball_base_url = self.app.settings['tarball-server']
-        repo_resolver = morphlib.repoaliasresolver.RepoAliasResolver(
-            self.app.settings['repo-alias'])
-        lrc = morphlib.localrepocache.LocalRepoCache(
-            self.app, cachedir, repo_resolver, tarball_base_url)
-
-        remote_url = morphlib.util.get_git_resolve_cache_server(
-            self.app.settings)
-        if remote_url:
-            rrc = morphlib.remoterepocache.RemoteRepoCache(
-                remote_url, repo_resolver)
-        else:
-            rrc = None
-            
         build_command = morphlib.buildcommand.BuildCommand(self.app)
 
         # traverse the morphs to list all the sources
diff --git a/morphlib/plugins/system_manifests_plugin.py b/morphlib/plugins/system_manifests_plugin.py
index 86388737..7fe33102 100644
--- a/morphlib/plugins/system_manifests_plugin.py
+++ b/morphlib/plugins/system_manifests_plugin.py
@@ -84,7 +84,7 @@ class SystemManifestsPlugin(cliapp.Plugin):
         system_filenames = map(morphlib.util.sanitise_morphology_path,
                                args[2:])
 
-        self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app)
+        self.repo_cache = morphlib.util.new_repo_cache(self.app)
         self.resolver = morphlib.artifactresolver.ArtifactResolver()
 
         for system_filename in system_filenames:
@@ -104,9 +104,7 @@ class SystemManifestsPlugin(cliapp.Plugin):
             msg='Creating source pool for %(system)s',
             system=system_filename, chatty=True)
         source_pool = morphlib.sourceresolver.create_source_pool(
-            self.lrc, self.rrc, repo, ref, [system_filename],
-            cachedir=self.app.settings['cachedir'],
-            update_repos = not self.app.settings['no-git-update'],
+            self.repo_cache, repo, ref, [system_filename],
             status_cb=self.app.status)
 
         self.app.status(
@@ -135,10 +133,11 @@ class SystemManifestsPlugin(cliapp.Plugin):
         except IndexError:
             trove_id = None
         with morphlib.util.temp_dir(dir=self.app.settings['tempdir']) as td:
-            lorries = get_lorry_repos(td, self.lrc, self.app.status, trove_id,
+            lorries = get_lorry_repos(td, self.repo_cache, self.app.status,
+                                      trove_id,
                                       self.app.settings['trove-host'])
             manifest = Manifest(system_artifact.name, td, self.app.status,
-                                self.lrc)
+                                self.repo_cache)
 
             old_prefix = self.app.status_prefix
             sources = set(a.source for a in system_artifact.walk()
@@ -150,7 +149,8 @@ class SystemManifestsPlugin(cliapp.Plugin):
                 name = source.morphology['name']
                 ref = source.original_ref
 
-                cached = self.lrc.get_updated_repo(source.repo_name, ref)
+                cached = self.repo_cache.get_updated_repo(source.repo_name,
+                                                          ref)
 
                 new_prefix = '[%d/%d][%s] ' % (i, len(sources), name)
                 self.app.status_prefix = old_prefix + new_prefix
@@ -169,8 +169,8 @@ def run_licensecheck(filename):
     else:
         return output[len(filename) + 2:].strip()
 
-def checkout_repo(lrc, repo, dest, ref='master'):
-    cached = lrc.get_updated_repo(repo, ref)
+def checkout_repo(repo_cache, repo, dest, ref='master'):
+    cached = repo_cache.get_updated_repo(repo, ref)
     if not os.path.exists(dest):
         morphlib.gitdir.checkout_from_cached_repo(repo, ref, dest)
 
@@ -235,14 +235,15 @@ def get_upstream_address(chunk_url, lorries, status):
                chunk=chunk_url)
         return 'UNKNOWN'
 
-def get_lorry_repos(tempdir, lrc, status, trove_id, trove_host):
+def get_lorry_repos(tempdir, repo_cache, status, trove_id, trove_host):
     lorries = []
     try:
         baserock_lorry_repo = 'baserock:local-config/lorries'
         lorrydir = os.path.join(tempdir, 'baserock-lorries')
-        baserock_lorrydir = checkout_repo(lrc, baserock_lorry_repo, lorrydir)
+        baserock_lorrydir = checkout_repo(repo_cache, baserock_lorry_repo,
+                                          lorrydir)
         lorries.extend(load_lorries(lorrydir))
-    except morphlib.localrepocache.NoRemote as e:
+    except morphlib.repocache.NoRemote as e:
         status(msg="WARNING: Could not find lorries from git.baserock.org, "
                    "expected to find them on %(trove)s at %(reponame)s",
                trove=trove_host, reponame = e.reponame)
@@ -252,9 +253,10 @@ def get_lorry_repos(tempdir, lrc, status, trove_id, trove_host):
             trove_lorry_repo =  ('http://%s/git/%s/local-config/lorries' %
                                  (trove_host, trove_id))
             lorrydir = os.path.join(tempdir, '%s-lorries' % trove_id)
-            trove_lorrydir = checkout_repo(lrc, trove_lorry_repo, lorrydir)
+            trove_lorrydir = checkout_repo(repo_cache, trove_lorry_repo,
+                                           lorrydir)
             lorries.extend(load_lorries(lorrydir))
-        except morphlib.localrepocache.NoRemote as e:
+        except morphlib.repocache.NoRemote as e:
             status(msg="WARNING: Could not find lorries repo on %(trove)s "
                        "at %(reponame)s",
                    trove=trove_host, reponame=e.reponame)
@@ -268,10 +270,10 @@ def get_lorry_repos(tempdir, lrc, status, trove_id, trove_host):
 class Manifest(object):
     """Writes out a manifest of what's included in a system."""
 
-    def __init__(self, system_name, tempdir, status_cb, lrc):
+    def __init__(self, system_name, tempdir, status_cb, repo_cache):
         self.tempdir = tempdir
         self.status = status_cb
-        self.lrc = lrc
+        self.repo_cache = repo_cache
         path = os.path.join(os.getcwd(), system_name + '-manifest.csv')
         self.status(msg='Creating %(path)s', path=path)
         self.file = open(path, 'wb')