diff options
| -rw-r--r-- | morphlib/app.py | 8 | ||||
| -rw-r--r-- | morphlib/artifactresolver.py | 75 | ||||
| -rw-r--r-- | morphlib/buildcommand.py | 26 | ||||
| -rw-r--r-- | morphlib/source.py | 52 | ||||
| -rw-r--r-- | morphlib/sourcepool.py | 13 | ||||
| -rw-r--r-- | morphlib/stagingarea.py | 4 |
6 files changed, 112 insertions, 66 deletions
diff --git a/morphlib/app.py b/morphlib/app.py index 88eb58a4..fae4c84b 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -290,9 +290,11 @@ class Morph(cliapp.Application): pool = morphlib.sourcepool.SourcePool() def add_to_pool(reponame, ref, filename, absref, tree, morphology): - source = morphlib.source.Source(reponame, ref, absref, tree, - morphology, filename) - pool.add(source) + sources = morphlib.source.make_sources(reponame, ref, + filename, absref, + tree, morphology) + for source in sources: + pool.add(source) self.traverse_morphs(repo, ref, [filename], lrc, rrc, update=not self.settings['no-git-update'], diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index 49e03664..a60a8989 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -68,7 +68,7 @@ class ArtifactResolver(object): while queue: source = queue.popleft() - if source.morphology['kind'] == 'system': + if source.morphology['kind'] == 'system': # pragma: no cover systems = [source.artifacts[name] for name in source.split_rules.artifacts] @@ -85,8 +85,11 @@ class ArtifactResolver(object): artifacts.append(artifact) self._added_artifacts.add(artifact) elif source.morphology['kind'] == 'stratum': + # Iterate split_rules.artifacts, rather than + # artifacts.values() to preserve ordering strata = [source.artifacts[name] - for name in source.split_rules.artifacts] + for name in source.split_rules.artifacts + if name in source.artifacts] # If we were not given systems, return the strata here, # rather than have the systems return them. @@ -126,26 +129,30 @@ class ArtifactResolver(object): if x.morphology['kind'] != 'chunk'] return collections.deque(sources) - def _resolve_system_dependencies(self, systems, source, queue): + def _resolve_system_dependencies(self, systems, + source, queue): # pragma: no cover artifacts = [] for info in source.morphology['strata']: - stratum_source = self._source_pool.lookup( + for stratum_source in self._source_pool.lookup( info.get('repo') or source.repo_name, info.get('ref') or source.original_ref, - morphlib.util.sanitise_morphology_path(info['morph'])) - stratum_name = stratum_source.morphology['name'] + morphlib.util.sanitise_morphology_path(info['morph'])): - matches, overlaps, unmatched = source.split_rules.partition( - ((stratum_name, sta_name) for sta_name - in stratum_source.split_rules.artifacts)) - for system in systems: - for (stratum_name, sta_name) in matches[system.name]: - stratum = stratum_source.artifacts[sta_name] - system.add_dependency(stratum) - artifacts.append(stratum) + stratum_morph_name = stratum_source.morphology['name'] + + matches, overlaps, unmatched = source.split_rules.partition( + ((stratum_morph_name, sta_name) for sta_name + in stratum_source.split_rules.artifacts)) + for system in systems: + for (stratum_name, sta_name) in matches[system.name]: + if sta_name in stratum_source.artifacts: + stratum_artifact = \ + stratum_source.artifacts[sta_name] + system.add_dependency(stratum_artifact) + artifacts.append(stratum_artifact) - queue.append(stratum_source) + queue.append(stratum_source) return artifacts @@ -155,28 +162,32 @@ class ArtifactResolver(object): stratum_build_depends = [] for stratum_info in source.morphology.get('build-depends') or []: - other_source = self._source_pool.lookup( + for other_source in self._source_pool.lookup( stratum_info.get('repo') or source.repo_name, stratum_info.get('ref') or source.original_ref, - morphlib.util.sanitise_morphology_path(stratum_info['morph'])) + morphlib.util.sanitise_morphology_path(stratum_info['morph'])): - # Make every stratum artifact this stratum source produces - # depend on every stratum artifact the other stratum source - # produces. - for sta_name in other_source.split_rules.artifacts: - other_stratum = other_source.artifacts[sta_name] + # Make every stratum artifact this stratum source produces + # depend on every stratum artifact the other stratum source + # produces. + for sta_name in other_source.split_rules.artifacts: + # Strata have split rules for artifacts they don't build, + # since they need to know to yield a match to its sibling + if sta_name not in other_source.artifacts: + continue + other_stratum = other_source.artifacts[sta_name] - stratum_build_depends.append(other_stratum) + stratum_build_depends.append(other_stratum) - artifacts.append(other_stratum) + artifacts.append(other_stratum) - for stratum in strata: - if other_stratum.depends_on(stratum): - raise MutualDependencyError(stratum, other_stratum) + for stratum in strata: + if other_stratum.depends_on(stratum): + raise MutualDependencyError(stratum, other_stratum) - stratum.add_dependency(other_stratum) + stratum.add_dependency(other_stratum) - queue.append(other_source) + queue.append(other_source) # 'name' here is the chunk artifact name name_to_processed_artifacts = {} @@ -187,9 +198,9 @@ class ArtifactResolver(object): chunk_source = self._source_pool.lookup( info['repo'], info['ref'], - filename) + filename)[0] - chunk_name = chunk_source.morphology['name'] + chunk_name = chunk_source.name # Resolve now to avoid a search for the parent morphology later chunk_source.build_mode = info['build-mode'] @@ -205,7 +216,7 @@ class ArtifactResolver(object): chunk_artifact.add_dependency(other_stratum) # Add dependencies between chunks mentioned in this stratum - for name in build_depends: + for name in build_depends: # pragma: no cover if name not in name_to_processed_artifacts: raise DependencyOrderError( source, info['name'], name) diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 352b43d2..436e23eb 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -208,7 +208,7 @@ class BuildCommand(object): # a build-dependency, then they must both have the same Repository # and Ref specified. if src.morphology['kind'] == 'stratum': - name = src.morphology['name'] + name = src.name ref = src.sha1[:7] self.app.status(msg='Stratum [%(name)s] version is %(ref)s', name=name, ref=ref) @@ -236,18 +236,18 @@ class BuildCommand(object): logging.debug( 'Validating cross ref to %s:%s:%s' % (repo_name, ref, filename)) - other = srcpool.lookup(repo_name, ref, filename) - if other.morphology['kind'] != wanted: - raise morphlib.Error( - '%s %s references %s:%s:%s which is a %s, ' - 'instead of a %s' % - (src.morphology['kind'], - src.morphology['name'], - repo_name, - ref, - filename, - other.morphology['kind'], - wanted)) + for other in srcpool.lookup(repo_name, ref, filename): + if other.morphology['kind'] != wanted: + raise morphlib.Error( + '%s %s references %s:%s:%s which is a %s, ' + 'instead of a %s' % + (src.morphology['kind'], + src.name, + repo_name, + ref, + filename, + other.morphology['kind'], + wanted)) def _find_root_artifacts(self, artifacts): '''Find all the root artifacts among a set of artifacts in a DAG. diff --git a/morphlib/source.py b/morphlib/source.py index d0f69a28..3d7e5a0f 100644 --- a/morphlib/source.py +++ b/morphlib/source.py @@ -35,8 +35,9 @@ class Source(object): ''' - def __init__(self, repo_name, original_ref, sha1, tree, morphology, - filename): + def __init__(self, name, repo_name, original_ref, sha1, tree, morphology, + filename, split_rules): + self.name = name self.repo = None self.repo_name = repo_name self.original_ref = original_ref @@ -45,17 +46,46 @@ class Source(object): self.morphology = morphology self.filename = filename - kind = morphology['kind'] - unifier = getattr(morphlib.artifactsplitrule, - 'unify_%s_matches' % kind) - self.split_rules = unifier(morphology) - self.artifacts = {name: morphlib.artifact.Artifact(self, name) - for name in self.split_rules.artifacts} + self.split_rules = split_rules + self.artifacts = None def __str__(self): # pragma: no cover - return '%s|%s|%s' % (self.repo_name, - self.original_ref, - self.filename) + return '%s|%s|%s|%s' % (self.repo_name, + self.original_ref, + self.filename, + self.name) def __repr__(self): # pragma: no cover return 'Source(%s)' % str(self) + + +def make_sources(reponame, ref, filename, absref, tree, morphology): + kind = morphology['kind'] + if kind in ('system', 'chunk'): + unifier = getattr(morphlib.artifactsplitrule, + 'unify_%s_matches' % kind) + split_rules = unifier(morphology) + # chunk and system sources are named after the morphology + source_name = morphology['name'] + source = morphlib.source.Source(source_name, reponame, ref, + absref, tree, morphology, + filename, split_rules) + source.artifacts = {name: morphlib.artifact.Artifact(source, name) + for name in split_rules.artifacts} + yield source + elif kind == 'stratum': # pragma: no cover + unifier = morphlib.artifactsplitrule.unify_stratum_matches + split_rules = unifier(morphology) + for name in split_rules.artifacts: + source = morphlib.source.Source( + name, # stratum source name is artifact name + reponame, ref, absref, tree, morphology, filename, + # stratum sources need to match the unified + # split rules, so they know to yield the match + # to a different source + split_rules) + source.artifacts = {name: morphlib.artifact.Artifact(source, name)} + yield source + else: + # cluster morphologies don't have sources + pass diff --git a/morphlib/sourcepool.py b/morphlib/sourcepool.py index ec134c0a..6dfcb2c3 100644 --- a/morphlib/sourcepool.py +++ b/morphlib/sourcepool.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,12 +14,15 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import collections + + class SourcePool(object): '''Manage a collection of Source objects.''' def __init__(self): - self._sources = {} + self._sources = collections.defaultdict(dict) self._order = [] def _key(self, repo_name, original_ref, filename): @@ -30,8 +33,8 @@ class SourcePool(object): key = self._key(source.repo_name, source.original_ref, source.filename) - if key not in self._sources: - self._sources[key] = source + if key not in self._sources or source.name not in self._sources[key]: + self._sources[key][source.name] = source self._order.append(source) def lookup(self, repo_name, original_ref, filename): @@ -42,7 +45,7 @@ class SourcePool(object): ''' key = self._key(repo_name, original_ref, filename) - return self._sources[key] + return self._sources[key].values() def __iter__(self): '''Iterate over sources in the pool, in the order they were added.''' diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index 0126b4d9..bfe0a716 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -65,8 +65,8 @@ class StagingArea(object): os.makedirs(dirname) def _dir_for_source(self, source, suffix): - basename = '%s.%s' % (str(source.morphology['name']), suffix) - dirname = os.path.join(self.dirname, basename) + dirname = os.path.join(self.dirname, + '%s.%s' % (str(source.name), suffix)) self._mkdir(dirname) return dirname |