From 2a9c29502a76a54aba3cc92c6e617abbe41ba1ee Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Thu, 23 Oct 2014 15:16:07 +0100 Subject: Move create_source_pool code into new 'sourceresolver' module This code is an essential part of 'morph build'. It's quite complex and really shouldn't be mixed in with the base Application class. Given a dedicated class we can store some state in the object and avoid functions with seven parameters, too. --- morphlib/__init__.py | 1 + morphlib/app.py | 131 ---------------------- morphlib/buildcommand.py | 7 +- morphlib/plugins/list_artifacts_plugin.py | 6 +- morphlib/sourceresolver.py | 176 ++++++++++++++++++++++++++++++ without-test-modules | 4 + 6 files changed, 189 insertions(+), 136 deletions(-) create mode 100644 morphlib/sourceresolver.py diff --git a/morphlib/__init__.py b/morphlib/__init__.py index f98c11aa..8319430c 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -79,6 +79,7 @@ import repoaliasresolver import savefile import source import sourcepool +import sourceresolver import stagingarea import stopwatch import sysbranchdir diff --git a/morphlib/app.py b/morphlib/app.py index c4d45aae..3f4171a4 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -15,13 +15,11 @@ import cliapp -import collections import logging import os import sys import time import urlparse -import warnings import extensions import morphlib @@ -290,135 +288,6 @@ class Morph(cliapp.Application): morphlib.util.sanitise_morphology_path(args[2])) args = args[3:] - def create_source_pool(self, lrc, rrc, repo, ref, filename, - original_ref=None): - pool = morphlib.sourcepool.SourcePool() - - def add_to_pool(reponame, ref, filename, absref, tree, morphology): - sources = morphlib.source.make_sources(reponame, ref, - filename, absref, - tree, morphology) - for source in sources: - pool.add(source) - - self.traverse_morphs(repo, ref, [filename], lrc, rrc, - update=not self.settings['no-git-update'], - visit=add_to_pool, - definitions_original_ref=original_ref) - return pool - - def resolve_ref(self, lrc, rrc, reponame, ref, update=True): - '''Resolves commit and tree sha1s of the ref in a repo and returns it. - - If update is True then this has the side-effect of updating - or cloning the repository into the local repo cache. - ''' - absref = None - - if lrc.has_repo(reponame): - repo = lrc.get_repo(reponame) - if update and repo.requires_update_for_ref(ref): - self.status(msg='Updating cached git repository %(reponame)s ' - 'for ref %(ref)s', reponame=reponame, ref=ref) - repo.update() - # If the user passed --no-git-update, and the ref is a SHA1 not - # available locally, this call will raise an exception. - absref, tree = repo.resolve_ref(ref) - elif rrc is not None: - try: - absref, tree = rrc.resolve_ref(reponame, ref) - if absref is not None: - self.status(msg='Resolved %(reponame)s %(ref)s via remote ' - 'repo cache', - reponame=reponame, - ref=ref, - chatty=True) - except BaseException, e: - logging.warning('Caught (and ignored) exception: %s' % str(e)) - if absref is None: - if update: - self.status(msg='Caching git repository %(reponame)s', - reponame=reponame) - repo = lrc.cache_repo(reponame) - repo.update() - else: - repo = lrc.get_repo(reponame) - absref, tree = repo.resolve_ref(ref) - return absref, tree - - def traverse_morphs(self, definitions_repo, definitions_ref, - system_filenames, lrc, rrc, update=True, - visit=lambda rn, rf, fn, arf, m: None, - definitions_original_ref=None): - morph_factory = morphlib.morphologyfactory.MorphologyFactory( - lrc, rrc, self.status) - definitions_queue = collections.deque(system_filenames) - chunk_in_definitions_repo_queue = [] - chunk_in_source_repo_queue = [] - resolved_refs = {} - resolved_morphologies = {} - - # Resolve the (repo, ref) pair for the definitions repo, cache result. - definitions_absref, definitions_tree = self.resolve_ref( - lrc, rrc, definitions_repo, definitions_ref, update) - - if definitions_original_ref: - definitions_ref = definitions_original_ref - - while definitions_queue: - filename = definitions_queue.popleft() - - key = (definitions_repo, definitions_absref, filename) - if not key in resolved_morphologies: - resolved_morphologies[key] = morph_factory.get_morphology(*key) - morphology = resolved_morphologies[key] - - visit(definitions_repo, definitions_ref, filename, - definitions_absref, definitions_tree, morphology) - if morphology['kind'] == 'cluster': - raise cliapp.AppException( - "Cannot build a morphology of type 'cluster'.") - elif morphology['kind'] == 'system': - definitions_queue.extend( - morphlib.util.sanitise_morphology_path(s['morph']) - for s in morphology['strata']) - elif morphology['kind'] == 'stratum': - if morphology['build-depends']: - definitions_queue.extend( - morphlib.util.sanitise_morphology_path(s['morph']) - for s in morphology['build-depends']) - for c in morphology['chunks']: - if 'morph' not in c: - path = morphlib.util.sanitise_morphology_path( - c.get('morph', c['name'])) - chunk_in_source_repo_queue.append( - (c['repo'], c['ref'], path)) - continue - chunk_in_definitions_repo_queue.append( - (c['repo'], c['ref'], c['morph'])) - - for repo, ref, filename in chunk_in_definitions_repo_queue: - if (repo, ref) not in resolved_refs: - resolved_refs[repo, ref] = self.resolve_ref( - lrc, rrc, repo, ref, update) - absref, tree = resolved_refs[repo, ref] - key = (definitions_repo, definitions_absref, filename) - if not key in resolved_morphologies: - resolved_morphologies[key] = morph_factory.get_morphology(*key) - morphology = resolved_morphologies[key] - visit(repo, ref, filename, absref, tree, morphology) - - for repo, ref, filename in chunk_in_source_repo_queue: - if (repo, ref) not in resolved_refs: - resolved_refs[repo, ref] = self.resolve_ref( - lrc, rrc, repo, ref, update) - absref, tree = resolved_refs[repo, ref] - key = (repo, absref, filename) - if key not in resolved_morphologies: - resolved_morphologies[key] = morph_factory.get_morphology(*key) - morphology = resolved_morphologies[key] - visit(repo, ref, filename, absref, tree, morphology) - def cache_repo_and_submodules(self, cache, url, ref, done): subs_to_process = set() subs_to_process.add((url, ref)) diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 438badb3..a658fc55 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -94,10 +94,11 @@ class BuildCommand(object): ''' self.app.status(msg='Creating source pool', chatty=True) - srcpool = self.app.create_source_pool( + srcpool = morphlib.sourceresolver.create_source_pool( self.lrc, self.rrc, repo_name, ref, filename, - original_ref=original_ref) - + original_ref=original_ref, + update_repos=not self.app.settings['no-git-update'], + status_cb=self.app.status) return srcpool def validate_sources(self, srcpool): diff --git a/morphlib/plugins/list_artifacts_plugin.py b/morphlib/plugins/list_artifacts_plugin.py index 0f14a579..713473ac 100644 --- a/morphlib/plugins/list_artifacts_plugin.py +++ b/morphlib/plugins/list_artifacts_plugin.py @@ -84,8 +84,10 @@ class ListArtifactsPlugin(cliapp.Plugin): self.app.status( msg='Creating source pool for %s' % system_filename, chatty=True) - source_pool = self.app.create_source_pool( - self.lrc, self.rrc, repo, ref, system_filename) + source_pool = morphlib.SourceResolver.create_source_pool( + self.lrc, self.rrc, repo, ref, system_filename, + update_repos = not self.app.settings['no-git-update'], + status_cb=self.app.status) self.app.status( msg='Resolving artifacts for %s' % system_filename, chatty=True) diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py new file mode 100644 index 00000000..dee16ea1 --- /dev/null +++ b/morphlib/sourceresolver.py @@ -0,0 +1,176 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp + +import collections +import logging + +import morphlib + + +class SourceResolver(object): + '''Provides a way of resolving the set of sources for a given system.''' + + def __init__(self, lrc, rrc, status_cb): + self.lrc = lrc + self.rrc = rrc + + self.status = status_cb + + def resolve_ref(self, reponame, ref, update=True): + '''Resolves commit and tree sha1s of the ref in a repo and returns it. + + If update is True then this has the side-effect of updating + or cloning the repository into the local repo cache. + ''' + absref = None + + if self.lrc.has_repo(reponame): + repo = self.lrc.get_repo(reponame) + if update and repo.requires_update_for_ref(ref): + self.status(msg='Updating cached git repository %(reponame)s ' + 'for ref %(ref)s', reponame=reponame, ref=ref) + repo.update() + # If the user passed --no-git-update, and the ref is a SHA1 not + # available locally, this call will raise an exception. + absref, tree = repo.resolve_ref(ref) + elif self.rrc is not None: + try: + absref, tree = self.rrc.resolve_ref(reponame, ref) + if absref is not None: + self.status(msg='Resolved %(reponame)s %(ref)s via remote ' + 'repo cache', + reponame=reponame, + ref=ref, + chatty=True) + except BaseException, e: + logging.warning('Caught (and ignored) exception: %s' % str(e)) + if absref is None: + if update: + self.status(msg='Caching git repository %(reponame)s', + reponame=reponame) + repo = self.lrc.cache_repo(reponame) + repo.update() + else: + repo = self.lrc.get_repo(reponame) + absref, tree = repo.resolve_ref(ref) + return absref, tree + + def traverse_morphs(self, definitions_repo, definitions_ref, + system_filenames, update=True, + visit=lambda rn, rf, fn, arf, m: None, + definitions_original_ref=None): + morph_factory = morphlib.morphologyfactory.MorphologyFactory( + self.lrc, self.rrc, self.status) + definitions_queue = collections.deque(system_filenames) + chunk_in_definitions_repo_queue = [] + chunk_in_source_repo_queue = [] + resolved_refs = {} + resolved_morphologies = {} + + # Resolve the (repo, ref) pair for the definitions repo, cache result. + definitions_absref, definitions_tree = self.resolve_ref( + definitions_repo, definitions_ref, update) + + if definitions_original_ref: + definitions_ref = definitions_original_ref + + while definitions_queue: + filename = definitions_queue.popleft() + + key = (definitions_repo, definitions_absref, filename) + if not key in resolved_morphologies: + resolved_morphologies[key] = morph_factory.get_morphology(*key) + morphology = resolved_morphologies[key] + + visit(definitions_repo, definitions_ref, filename, + definitions_absref, definitions_tree, morphology) + if morphology['kind'] == 'cluster': + raise cliapp.AppException( + "Cannot build a morphology of type 'cluster'.") + elif morphology['kind'] == 'system': + definitions_queue.extend( + morphlib.util.sanitise_morphology_path(s['morph']) + for s in morphology['strata']) + elif morphology['kind'] == 'stratum': + if morphology['build-depends']: + definitions_queue.extend( + morphlib.util.sanitise_morphology_path(s['morph']) + for s in morphology['build-depends']) + for c in morphology['chunks']: + if 'morph' not in c: + path = morphlib.util.sanitise_morphology_path( + c.get('morph', c['name'])) + chunk_in_source_repo_queue.append( + (c['repo'], c['ref'], path)) + continue + chunk_in_definitions_repo_queue.append( + (c['repo'], c['ref'], c['morph'])) + + for repo, ref, filename in chunk_in_definitions_repo_queue: + if (repo, ref) not in resolved_refs: + resolved_refs[repo, ref] = self.resolve_ref(repo, ref, update) + absref, tree = resolved_refs[repo, ref] + key = (definitions_repo, definitions_absref, filename) + if not key in resolved_morphologies: + resolved_morphologies[key] = morph_factory.get_morphology(*key) + morphology = resolved_morphologies[key] + visit(repo, ref, filename, absref, tree, morphology) + + for repo, ref, filename in chunk_in_source_repo_queue: + if (repo, ref) not in resolved_refs: + resolved_refs[repo, ref] = self.resolve_ref(repo, ref, update) + absref, tree = resolved_refs[repo, ref] + key = (repo, absref, filename) + if key not in resolved_morphologies: + resolved_morphologies[key] = morph_factory.get_morphology(*key) + morphology = resolved_morphologies[key] + visit(repo, ref, filename, absref, tree, morphology) + + +def create_source_pool(lrc, rrc, repo, ref, filename, + original_ref=None, update_repos=True, + status_cb=None): + '''Find all the sources involved in building a given system. + + Given a system morphology, this function will traverse the tree of stratum + and chunk morphologies that the system points to and create appropriate + Source objects. These are added to a new SourcePool object, which is + returned. + + Note that Git submodules are not considered 'sources' in the current + implementation, and so they must be handled separately. + + The 'lrc' and 'rrc' parameters specify the local and remote Git repository + caches used for resolving the sources. + + ''' + pool = morphlib.sourcepool.SourcePool() + + def add_to_pool(reponame, ref, filename, absref, tree, morphology): + sources = morphlib.source.make_sources(reponame, ref, + filename, absref, + tree, morphology) + for source in sources: + pool.add(source) + + resolver = SourceResolver(lrc, rrc, status_cb) + resolver.traverse_morphs(repo, ref, [filename], + update=update_repos, + visit=add_to_pool, + definitions_original_ref=original_ref) + return pool diff --git a/without-test-modules b/without-test-modules index 55e5291d..530deb4f 100644 --- a/without-test-modules +++ b/without-test-modules @@ -52,3 +52,7 @@ distbuild/timer_event_source.py distbuild/worker_build_scheduler.py # Not unit tested, since it needs a full system branch morphlib/buildbranch.py + +# Requires rather a lot of fake data in order to be unit tested; better to +# leave it to the functional tests. +morphlib/sourceresolver.py -- cgit v1.2.1