From 667158cb9b0f2110c96135d8b60cc789dd790d53 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 6 May 2015 17:24:55 +0000 Subject: FIXUP: WIP --- morphlib/__init__.py | 1 + morphlib/buildcommand.py | 9 ++++-- morphlib/cmdline_parse_utils.py | 5 +-- morphlib/plugins/diff_plugin.py | 69 +++++++++++++++++++++++++++++++++++------ morphlib/sourceresolver.py | 6 ++-- 5 files changed, 71 insertions(+), 19 deletions(-) diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 90cc3d80..94b42580 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -59,6 +59,7 @@ import buildsystem import builder import cachedrepo import cachekeycomputer +import cmdline_parse_utils import extensions import extractedtarball import fsutils diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index efd10f26..9ebe75d4 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -84,7 +84,8 @@ class BuildCommand(object): return morphlib.buildenvironment.BuildEnvironment(self.app.settings, arch) - def create_source_pool(self, repo_name, ref, filename, original_ref=None): + def create_source_pool(self, repo_name, ref, filename=None, + original_ref=None, filenames=None): '''Find the source objects required for building a the given artifact The SourcePool will contain every stratum and chunk dependency of the @@ -93,12 +94,14 @@ class BuildCommand(object): ''' self.app.status(msg='Creating source pool', chatty=True) + if filenames is None and filenames is not None: + filenames = (filename,) srcpool = morphlib.sourceresolver.create_source_pool( - self.lrc, self.rrc, repo_name, ref, filename, + self.lrc, self.rrc, repo_name, ref, filename=None, cachedir=self.app.settings['cachedir'], original_ref=original_ref, update_repos=not self.app.settings['no-git-update'], - status_cb=self.app.status) + status_cb=self.app.status, filenames=filenames) return srcpool def validate_sources(self, srcpool): diff --git a/morphlib/cmdline_parse_utils.py b/morphlib/cmdline_parse_utils.py index c5e08ec1..c1f69e1b 100644 --- a/morphlib/cmdline_parse_utils.py +++ b/morphlib/cmdline_parse_utils.py @@ -14,13 +14,10 @@ # with this program. If not, see . -from itertools import groupby as _groupby -from itertools import takewhile as _takewhile - from cliapp import AppException as _AppException -from .utils import word_join_list as _word_join_list +from .util import word_join_list as _word_join_list def _split(iterable, split_token): diff --git a/morphlib/plugins/diff_plugin.py b/morphlib/plugins/diff_plugin.py index 4763ba40..1f587082 100644 --- a/morphlib/plugins/diff_plugin.py +++ b/morphlib/plugins/diff_plugin.py @@ -14,11 +14,15 @@ # with this program. If not, see . +from collections import defaultdict + import cliapp -from morphlib.cmdline_parse_utils import parse_cmdline_system_lists +from morphlib.buildcommand import BuildCommand +from morphlib.cmdline_parse_utils import parse_definition_lists from morphlib.morphologyfinder import MorphologyFinder from morphlib.morphloader import MorphologyLoader +from morphlib.morphset import MorphologySet from morphlib.util import new_repo_caches @@ -57,19 +61,64 @@ class DiffPlugin(cliapp.Plugin): definition_list_name_list=('from', 'to'), args=args) - lrc, rrc = new_repo_caches(self.app) - ml = MorphologyLoader() - + bc = BuildCommand(self.app) - def load_morphset((reponame, ref, definitions)): - repo = lrc.get_updated_repo(reponame, ref=ref) + def get_systems((reponame, ref, definitions)): + 'Convert a definition path list into a list of systems' + ml = MorphologyLoader() + repo = bc.lrc.get_updated_repo(reponame, ref=ref) mf = MorphologyFinder(gitdir=repo.gitdir, ref=ref) + # We may have been given an empty set of definitions as input, in + # which case we instead use every we find. if not definitions: definitions = mf.list_morphologies() - ms = MorphologySet() + system_paths = set() for definition in definitions: m = ml.parse_morphology_text(mf.read_morphology(definition), definition) - ms.add_morphology(m) - from_morphset = load_morphset(from_spec) - to_morphset = load_morphset(to_spec) + if m.get('kind') == 'system' or 'strata' in m: + system_paths.add(definition) + return reponame, ref, system_paths + from_repo, from_ref, from_paths = get_systems(from_spec) + to_repo, to_ref, to_paths = get_systems(to_spec) + + # This abuses Sources because they conflate the ideas of "logical + # structure of the input of what we want to build" and the structure of + # what the output of that would be. + # If we were to pass the produced source pools to the artifact graphing + # code then they would become an invalid structure of the output, + # because the graphing code cannot handle multiple systems. + from_sp = bc.create_source_pool(repo_name=from_repo, ref=from_ref, + filenames=from_paths) + to_sp = bc.create_source_pool(repo_name=to_repo, ref=to_ref, + filenames=to_paths) + + from_by_name = defaultdict(set) + to_by_name = defaultdict(set) + for from_source in from_sp: + from_by_name[from_source.morphology['name']].add(from_source) + for to_source in to_sp: + to_by_name[to_source.morphology['name']].add(to_source) + + for name, from_sources in from_by_name.iteritems(): + from_source = iter(from_sources).next() + to_sources = to_by_name[name] + if not to_sources: + print('{} was removed'.format(name)) + continue + # We don't actually care that multiple sources match that + # lookup because of artifact splitting, so we'll take the + # first. + to_source = iter(to_sources).next() + if from_source.repo_name != to_source.repo_name: + print('{} repo changed from {} to {}'.format( + name, from_source.repo_name, to_source.repo_name)) + if from_source.original_ref != to_source.original_ref: + from_repo, to_repo = (bc.lrc.get_updated_repo(s.repo_name, + ref=s.sha1) + for s in (from_source, to_source)) + from_desc = from_repo.gitdir.version_guess(from_source.sha1) + to_desc = to_repo.gitdir.version_guess(to_source.sha1) + print('{} ref changed from {} to {}'.format( + from_source.morphology['name'], from_desc, to_desc)) + diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index e338dd4d..b8907bfe 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -600,7 +600,7 @@ class SourceResolver(object): def create_source_pool(lrc, rrc, repo, ref, filename, cachedir, original_ref=None, update_repos=True, - status_cb=None): # pragma: no cover + status_cb=None, filenames=None): # pragma: no cover '''Find all the sources involved in building a given system. Given a system morphology, this function will traverse the tree of stratum @@ -616,6 +616,8 @@ def create_source_pool(lrc, rrc, repo, ref, filename, cachedir, ''' pool = morphlib.sourcepool.SourcePool() + if filenames is None and filenames is not None: + filenames = (filename,) def add_to_pool(reponame, ref, filename, absref, tree, morphology): sources = morphlib.source.make_sources(reponame, ref, @@ -634,7 +636,7 @@ def create_source_pool(lrc, rrc, repo, ref, filename, cachedir, resolver = SourceResolver(lrc, rrc, tree_cache_manager, buildsystem_cache_manager, update_repos, status_cb) - resolver.traverse_morphs(repo, ref, [filename], + resolver.traverse_morphs(repo, ref, filenames, visit=add_to_pool, definitions_original_ref=original_ref) return pool -- cgit v1.2.1