From d58d8e8f7a4ec03ff14021a4515c8283dad52573 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Wed, 2 Mar 2016 17:11:34 +0000 Subject: Unify local and remote repo cache modules There's not really any reason you'd want to use the RemoteRepoCache class except as a workaround for the slow speed of some LocalRepoCache operations, so I can't see this ruining anyone's day. The main reason for doing this is so we can simply the sourceresolver code. One reason that the sourceresolver class is so hopelessly complicated is that it right now has to use two incompatible interfaces for Git repo caches. I've taken the opportunity to detangle the RepoCache class from the App class. Now all of the configuration for the RepoCache class is passed into the constructor explicitly. This makes the class usable from outside Morph: resolver = morphlib.repoaliasresolver.RepoAliasResolver(aliases=[]) repo_cache = morphlib.repocache.RepoCache('/src/cache/gits', resolver) Change-Id: I596c81d7645b67504c88e555172a8c238f4f8a66 --- morphlib/plugins/anchor_plugin.py | 5 ++-- morphlib/plugins/artifact_inspection_plugin.py | 37 ++++++++++---------------- morphlib/plugins/certify_plugin.py | 8 +++--- morphlib/plugins/cross-bootstrap_plugin.py | 2 +- morphlib/plugins/diff_plugin.py | 10 +++---- morphlib/plugins/get_repo_plugin.py | 6 ++--- morphlib/plugins/list_artifacts_plugin.py | 8 +++--- morphlib/plugins/show_dependencies_plugin.py | 20 +------------- morphlib/plugins/system_manifests_plugin.py | 34 ++++++++++++----------- 9 files changed, 50 insertions(+), 80 deletions(-) (limited to 'morphlib/plugins') diff --git a/morphlib/plugins/anchor_plugin.py b/morphlib/plugins/anchor_plugin.py index 7465c479..a9d07b39 100644 --- a/morphlib/plugins/anchor_plugin.py +++ b/morphlib/plugins/anchor_plugin.py @@ -137,9 +137,8 @@ class AnchorPlugin(cliapp.Plugin): for reponame, sources in sources_by_reponame.iteritems(): # UGLY HACK we need to push *FROM* our local repo cache to # avoid cloning everything multiple times. - repo = bc.lrc.get_updated_repo(reponame, - refs=(s.original_ref - for s in sources)) + repo = bc.repo_cache.get_updated_repo( + reponame, refs=(s.original_ref for s in sources)) remote = Remote(repo) push_url = resolver.push_url(reponame) diff --git a/morphlib/plugins/artifact_inspection_plugin.py b/morphlib/plugins/artifact_inspection_plugin.py index 413a0072..d396f93b 100644 --- a/morphlib/plugins/artifact_inspection_plugin.py +++ b/morphlib/plugins/artifact_inspection_plugin.py @@ -36,28 +36,23 @@ class NotASystemArtifactError(cliapp.AppException): class ProjectVersionGuesser(object): - def __init__(self, app, lrc, rrc, interesting_files): + def __init__(self, app, repo_cache, interesting_files): self.app = app - self.lrc = lrc - self.rrc = rrc + self.repo_cache = repo_cache self.interesting_files = interesting_files def file_contents(self, repo, ref, tree): filenames = [x for x in self.interesting_files if x in tree] - if filenames: - if self.lrc.has_repo(repo): - repository = self.lrc.get_updated_repo(repo, ref) - for filename in filenames: - yield filename, repository.read_file(filename, ref) - elif self.rrc: - for filename in filenames: - yield filename, self.rrc.cat_file(repo, ref, filename) + for filename in filenames: + # This can use a remote repo cache if available, to avoid having + # to clone every repo locally. + yield filename, self.repo_cache.cat_file(repo, ref, filename) class AutotoolsVersionGuesser(ProjectVersionGuesser): - def __init__(self, app, lrc, rrc): - ProjectVersionGuesser.__init__(self, app, lrc, rrc, [ + def __init__(self, app, repo_cache): + ProjectVersionGuesser.__init__(self, app, repo_cache, [ 'configure.ac', 'configure.in', 'configure.ac.in', @@ -136,9 +131,9 @@ class VersionGuesser(object): def __init__(self, app): self.app = app - self.lrc, self.rrc = morphlib.util.new_repo_caches(app) + self.repo_cache = morphlib.util.new_repo_cache(app) self.guessers = [ - AutotoolsVersionGuesser(app, self.lrc, self.rrc) + AutotoolsVersionGuesser(app, self.repo_cache) ] def guess_version(self, repo, ref): @@ -146,14 +141,10 @@ class VersionGuesser(object): repo=repo, ref=ref, chatty=True) version = None try: - if self.lrc.has_repo(repo): - repository = self.lrc.get_updated_repo(repo, ref) - tree = repository.list_files(ref=ref, recurse=False) - elif self.rrc: - repository = None - tree = self.rrc.ls_tree(repo, ref) - else: - return None + # This can use a remote repo cache if available, to avoid having + # to clone every repo locally. + tree = self.repo_cache.ls_tree(repo, ref) + for guesser in self.guessers: version = guesser.guess_version(repo, ref, tree) if version: diff --git a/morphlib/plugins/certify_plugin.py b/morphlib/plugins/certify_plugin.py index 735d0332..72d24a51 100644 --- a/morphlib/plugins/certify_plugin.py +++ b/morphlib/plugins/certify_plugin.py @@ -57,7 +57,7 @@ class CertifyPlugin(cliapp.Plugin): system_filenames = map(morphlib.util.sanitise_morphology_path, args[2:]) - self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) + self.repo_cache = morphlib.util.new_repo_cache(self.app) self.resolver = morphlib.artifactresolver.ArtifactResolver() for system_filename in system_filenames: @@ -69,9 +69,7 @@ class CertifyPlugin(cliapp.Plugin): self.app.status( msg='Creating source pool for %s' % system_filename, chatty=True) source_pool = morphlib.sourceresolver.create_source_pool( - self.lrc, self.rrc, repo, ref, [system_filename], - cachedir=self.app.settings['cachedir'], - update_repos = not self.app.settings['no-git-update'], + self.repo_cache, repo, ref, [system_filename], status_cb=self.app.status) self.app.status( @@ -115,7 +113,7 @@ class CertifyPlugin(cliapp.Plugin): .format(name, ref)) certified = False - cached = self.lrc.get_updated_repo(source.repo_name, ref) + cached = self.repo_cache.get_updated_repo(source.repo_name, ref) # Test that sha1 ref is anchored in a tag or branch, # and thus not a candidate for removal on `git gc`. diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py index 273e677d..8b8fbb2d 100644 --- a/morphlib/plugins/cross-bootstrap_plugin.py +++ b/morphlib/plugins/cross-bootstrap_plugin.py @@ -304,7 +304,7 @@ class CrossBootstrapPlugin(cliapp.Plugin): system_artifact.source, build_env, use_chroot=False) builder = BootstrapSystemBuilder( self.app, staging_area, build_command.lac, build_command.rac, - system_artifact.source, build_command.lrc, 1, False) + system_artifact.source, build_command.repo_cache, 1, False) builder.build_and_cache() self.app.status( diff --git a/morphlib/plugins/diff_plugin.py b/morphlib/plugins/diff_plugin.py index 26964df8..24a6d69a 100644 --- a/morphlib/plugins/diff_plugin.py +++ b/morphlib/plugins/diff_plugin.py @@ -22,7 +22,6 @@ from morphlib.cmdline_parse_utils import (definition_lists_synopsis, from morphlib.morphologyfinder import MorphologyFinder from morphlib.morphloader import MorphologyLoader from morphlib.morphset import MorphologySet -from morphlib.util import new_repo_caches class DiffPlugin(cliapp.Plugin): @@ -60,9 +59,10 @@ class DiffPlugin(cliapp.Plugin): name, from_source.repo_name, to_source.repo_name)) if from_source.original_ref != to_source.original_ref: - from_repo, to_repo = (self.bc.lrc.get_updated_repo(s.repo_name, - ref=s.sha1) - for s in (from_source, to_source)) + repo_cache = self.bc.repo_cache + from_repo, to_repo = (repo_cache.get_updated_repo(s.repo_name, + ref=s.sha1) + for s in (from_source, to_source)) from_desc = from_repo.version_guess(from_source.sha1) to_desc = to_repo.version_guess(to_source.sha1) @@ -100,7 +100,7 @@ class DiffPlugin(cliapp.Plugin): def get_systems((reponame, ref, definitions)): 'Convert a definition path list into a list of systems' ml = MorphologyLoader() - repo = self.bc.lrc.get_updated_repo(reponame, ref=ref) + repo = self.bc.repo_cache.get_updated_repo(reponame, ref=ref) mf = MorphologyFinder(gitdir=repo, ref=ref) # We may have been given an empty set of definitions as input, in # which case we instead use every we find. diff --git a/morphlib/plugins/get_repo_plugin.py b/morphlib/plugins/get_repo_plugin.py index fc81d6e5..ce0b7af0 100644 --- a/morphlib/plugins/get_repo_plugin.py +++ b/morphlib/plugins/get_repo_plugin.py @@ -101,9 +101,9 @@ class GetRepoPlugin(cliapp.Plugin): '%(stratum)s stratum', ref=ref or chunk_spec['ref'], chunk=chunk_spec['name'], stratum=morph['name']) - lrc, rrc = morphlib.util.new_repo_caches(self.app) - cached_repo = lrc.get_updated_repo(chunk_spec['repo'], - chunk_spec['ref']) + repo_cache = morphlib.util.new_repo_cache(self.app) + cached_repo = repo_cache.get_updated_repo(chunk_spec['repo'], + chunk_spec['ref']) try: self._clone_repo(cached_repo, dirname, diff --git a/morphlib/plugins/list_artifacts_plugin.py b/morphlib/plugins/list_artifacts_plugin.py index c2e6b459..2c098c2a 100644 --- a/morphlib/plugins/list_artifacts_plugin.py +++ b/morphlib/plugins/list_artifacts_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2014-2015 Codethink Limited +# Copyright (C) 2014-2016 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -58,7 +58,7 @@ class ListArtifactsPlugin(cliapp.Plugin): system_filenames = map(morphlib.util.sanitise_morphology_path, args[2:]) - self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) + self.repo_cache = morphlib.util.new_repo_cache(self.app) self.resolver = morphlib.artifactresolver.ArtifactResolver() artifact_files = set() @@ -85,9 +85,7 @@ class ListArtifactsPlugin(cliapp.Plugin): self.app.status( msg='Creating source pool for %s' % system_filename, chatty=True) source_pool = morphlib.sourceresolver.create_source_pool( - self.lrc, self.rrc, repo, ref, [system_filename], - cachedir=self.app.settings['cachedir'], - update_repos = not self.app.settings['no-git-update'], + self.repo_cache, repo, ref, [system_filename], status_cb=self.app.status) self.app.status( diff --git a/morphlib/plugins/show_dependencies_plugin.py b/morphlib/plugins/show_dependencies_plugin.py index 42f8f273..bfe4d6c2 100644 --- a/morphlib/plugins/show_dependencies_plugin.py +++ b/morphlib/plugins/show_dependencies_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2015 Codethink Limited +# Copyright (C) 2012-2016 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -42,24 +42,6 @@ class ShowDependenciesPlugin(cliapp.Plugin): of build dependencies of the constituent components. ''' - - if not os.path.exists(self.app.settings['cachedir']): - os.mkdir(self.app.settings['cachedir']) - cachedir = os.path.join(self.app.settings['cachedir'], 'gits') - tarball_base_url = self.app.settings['tarball-server'] - repo_resolver = morphlib.repoaliasresolver.RepoAliasResolver( - self.app.settings['repo-alias']) - lrc = morphlib.localrepocache.LocalRepoCache( - self.app, cachedir, repo_resolver, tarball_base_url) - - remote_url = morphlib.util.get_git_resolve_cache_server( - self.app.settings) - if remote_url: - rrc = morphlib.remoterepocache.RemoteRepoCache( - remote_url, repo_resolver) - else: - rrc = None - build_command = morphlib.buildcommand.BuildCommand(self.app) # traverse the morphs to list all the sources diff --git a/morphlib/plugins/system_manifests_plugin.py b/morphlib/plugins/system_manifests_plugin.py index 86388737..7fe33102 100644 --- a/morphlib/plugins/system_manifests_plugin.py +++ b/morphlib/plugins/system_manifests_plugin.py @@ -84,7 +84,7 @@ class SystemManifestsPlugin(cliapp.Plugin): system_filenames = map(morphlib.util.sanitise_morphology_path, args[2:]) - self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) + self.repo_cache = morphlib.util.new_repo_cache(self.app) self.resolver = morphlib.artifactresolver.ArtifactResolver() for system_filename in system_filenames: @@ -104,9 +104,7 @@ class SystemManifestsPlugin(cliapp.Plugin): msg='Creating source pool for %(system)s', system=system_filename, chatty=True) source_pool = morphlib.sourceresolver.create_source_pool( - self.lrc, self.rrc, repo, ref, [system_filename], - cachedir=self.app.settings['cachedir'], - update_repos = not self.app.settings['no-git-update'], + self.repo_cache, repo, ref, [system_filename], status_cb=self.app.status) self.app.status( @@ -135,10 +133,11 @@ class SystemManifestsPlugin(cliapp.Plugin): except IndexError: trove_id = None with morphlib.util.temp_dir(dir=self.app.settings['tempdir']) as td: - lorries = get_lorry_repos(td, self.lrc, self.app.status, trove_id, + lorries = get_lorry_repos(td, self.repo_cache, self.app.status, + trove_id, self.app.settings['trove-host']) manifest = Manifest(system_artifact.name, td, self.app.status, - self.lrc) + self.repo_cache) old_prefix = self.app.status_prefix sources = set(a.source for a in system_artifact.walk() @@ -150,7 +149,8 @@ class SystemManifestsPlugin(cliapp.Plugin): name = source.morphology['name'] ref = source.original_ref - cached = self.lrc.get_updated_repo(source.repo_name, ref) + cached = self.repo_cache.get_updated_repo(source.repo_name, + ref) new_prefix = '[%d/%d][%s] ' % (i, len(sources), name) self.app.status_prefix = old_prefix + new_prefix @@ -169,8 +169,8 @@ def run_licensecheck(filename): else: return output[len(filename) + 2:].strip() -def checkout_repo(lrc, repo, dest, ref='master'): - cached = lrc.get_updated_repo(repo, ref) +def checkout_repo(repo_cache, repo, dest, ref='master'): + cached = repo_cache.get_updated_repo(repo, ref) if not os.path.exists(dest): morphlib.gitdir.checkout_from_cached_repo(repo, ref, dest) @@ -235,14 +235,15 @@ def get_upstream_address(chunk_url, lorries, status): chunk=chunk_url) return 'UNKNOWN' -def get_lorry_repos(tempdir, lrc, status, trove_id, trove_host): +def get_lorry_repos(tempdir, repo_cache, status, trove_id, trove_host): lorries = [] try: baserock_lorry_repo = 'baserock:local-config/lorries' lorrydir = os.path.join(tempdir, 'baserock-lorries') - baserock_lorrydir = checkout_repo(lrc, baserock_lorry_repo, lorrydir) + baserock_lorrydir = checkout_repo(repo_cache, baserock_lorry_repo, + lorrydir) lorries.extend(load_lorries(lorrydir)) - except morphlib.localrepocache.NoRemote as e: + except morphlib.repocache.NoRemote as e: status(msg="WARNING: Could not find lorries from git.baserock.org, " "expected to find them on %(trove)s at %(reponame)s", trove=trove_host, reponame = e.reponame) @@ -252,9 +253,10 @@ def get_lorry_repos(tempdir, lrc, status, trove_id, trove_host): trove_lorry_repo = ('http://%s/git/%s/local-config/lorries' % (trove_host, trove_id)) lorrydir = os.path.join(tempdir, '%s-lorries' % trove_id) - trove_lorrydir = checkout_repo(lrc, trove_lorry_repo, lorrydir) + trove_lorrydir = checkout_repo(repo_cache, trove_lorry_repo, + lorrydir) lorries.extend(load_lorries(lorrydir)) - except morphlib.localrepocache.NoRemote as e: + except morphlib.repocache.NoRemote as e: status(msg="WARNING: Could not find lorries repo on %(trove)s " "at %(reponame)s", trove=trove_host, reponame=e.reponame) @@ -268,10 +270,10 @@ def get_lorry_repos(tempdir, lrc, status, trove_id, trove_host): class Manifest(object): """Writes out a manifest of what's included in a system.""" - def __init__(self, system_name, tempdir, status_cb, lrc): + def __init__(self, system_name, tempdir, status_cb, repo_cache): self.tempdir = tempdir self.status = status_cb - self.lrc = lrc + self.repo_cache = repo_cache path = os.path.join(os.getcwd(), system_name + '-manifest.csv') self.status(msg='Creating %(path)s', path=path) self.file = open(path, 'wb') -- cgit v1.2.1