diff options
-rw-r--r-- | morphlib/artifactresolver.py | 191 | ||||
-rw-r--r-- | morphlib/artifactresolver_tests.py | 592 |
2 files changed, 256 insertions, 527 deletions
diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index e64d458b..ae0cfcf5 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -16,6 +16,7 @@ import cliapp import collections +import logging import morphlib @@ -29,35 +30,21 @@ class MutualDependencyError(cliapp.AppException): class DependencyOrderError(cliapp.AppException): - def __init__(self, stratum, chunk, dependency_name): + def __init__(self, stratum_source, chunk, dependency_name): cliapp.AppException.__init__( self, 'In stratum %s, chunk %s references its dependency %s ' 'before it is defined' % - (stratum.source, chunk, dependency_name)) + (stratum_source, chunk, dependency_name)) class DependencyFormatError(cliapp.AppException): - def __init__(self, stratum, chunk): + def __init__(self, stratum_source, chunk): cliapp.AppException.__init__( self, 'In stratum %s, chunk %s uses an invalid ' - 'build-depends format' % (stratum.source, chunk)) + 'build-depends format' % (stratum_source, chunk)) -class UndefinedChunkArtifactError(cliapp.AppException): - - '''Exception raised when non-existent artifacts are referenced. - - Usually, this will only occur when a stratum refers to a chunk - artifact that is not defined in a chunk. - - ''' - - def __init__(self, parent, reference): - cliapp.AppException.__init__( - self, 'Undefined chunk artifact "%s" referenced in ' - 'stratum %s' % (reference, parent)) - class ArtifactResolver(object): @@ -91,12 +78,13 @@ class ArtifactResolver(object): source = queue.popleft() if source.morphology['kind'] == 'system': - systems = [source.artifacts[a] - for a in source.morphology.builds_artifacts] + systems = [source.artifacts[name] + for name in source.split_rules.artifacts] - if any(a not in self._added_artifacts for a in systems): - artifacts.extend(systems) - self._added_artifacts.update(systems) + for system in (s for s in systems + if s not in self._added_artifacts): + artifacts.append(system) + self._added_artifacts.add(system) resolved_artifacts = self._resolve_system_dependencies( systems, source, queue) @@ -106,28 +94,36 @@ class ArtifactResolver(object): artifacts.append(artifact) self._added_artifacts.add(artifact) elif source.morphology['kind'] == 'stratum': - assert len(source.morphology.builds_artifacts) == 1 - artifact = source.artifacts[ - source.morphology.builds_artifacts[0]] - - if not artifact in self._added_artifacts: - artifacts.append(artifact) - self._added_artifacts.add(artifact) + strata = [source.artifacts[name] + for name in source.split_rules.artifacts] + + # If we were not given systems, return the strata here, + # rather than have the systems return them. + if not any(s.morphology['kind'] == 'system' + for s in self._source_pool): + for stratum in (s for s in strata + if s not in self._added_artifacts): + artifacts.append(stratum) + self._added_artifacts.add(stratum) resolved_artifacts = self._resolve_stratum_dependencies( - artifact, queue) + strata, source, queue) for artifact in resolved_artifacts: if not artifact in self._added_artifacts: artifacts.append(artifact) self._added_artifacts.add(artifact) elif source.morphology['kind'] == 'chunk': - names = source.morphology.builds_artifacts - for name in names: - artifact = source.artifacts[name] - if not artifact in self._added_artifacts: - artifacts.append(artifact) - self._added_artifacts.add(artifact) + chunks = [source.artifacts[name] + for name in source.split_rules.artifacts] + # If we were only given chunks, return them here, rather than + # have the strata return them. + if not any(s.morphology['kind'] == 'stratum' + for s in self._source_pool): + for chunk in (c for c in chunks + if c not in self._added_artifacts): + artifacts.append(chunk) + self._added_artifacts.add(chunk) return artifacts @@ -147,67 +143,60 @@ class ArtifactResolver(object): info['repo'] or source.repo_name, info['ref'] or source.original_ref, '%s.morph' % info['morph']) + stratum_name = stratum_source.morphology['name'] - stratum_name = stratum_source.morphology.builds_artifacts[0] - stratum = stratum_source.artifacts[stratum_name] - + matches, overlaps, unmatched = source.split_rules.partition( + ((stratum_name, sta_name) for sta_name + in stratum_source.split_rules.artifacts)) for system in systems: - system.add_dependency(stratum) - queue.append(stratum_source) + for (stratum_name, sta_name) in matches[system.name]: + stratum = stratum_source.artifacts[sta_name] + system.add_dependency(stratum) + artifacts.append(stratum) - artifacts.append(stratum) + queue.append(stratum_source) return artifacts - def _resolve_stratum_dependencies(self, stratum, queue): + def _resolve_stratum_dependencies(self, strata, source, queue): artifacts = [] - strata = [] + stratum_build_depends = [] - if stratum.source.morphology['build-depends']: - for stratum_info in stratum.source.morphology['build-depends']: - other_source = self._source_pool.lookup( - stratum_info['repo'] or stratum.source.repo_name, - stratum_info['ref'] or stratum.source.original_ref, - '%s.morph' % stratum_info['morph']) + for stratum_info in source.morphology.get('build-depends') or []: + other_source = self._source_pool.lookup( + stratum_info['repo'] or source.repo_name, + stratum_info['ref'] or source.original_ref, + '%s.morph' % stratum_info['morph']) - other_stratum = other_source.artifacts[ - other_source.morphology.builds_artifacts[0]] + # Make every stratum artifact this stratum source produces + # depend on every stratum artifact the other stratum source + # produces. + for sta_name in other_source.split_rules.artifacts: + other_stratum = other_source.artifacts[sta_name] - strata.append(other_stratum) + stratum_build_depends.append(other_stratum) artifacts.append(other_stratum) - if other_stratum.depends_on(stratum): - raise MutualDependencyError(stratum, other_stratum) + for stratum in strata: + if other_stratum.depends_on(stratum): + raise MutualDependencyError(stratum, other_stratum) - stratum.add_dependency(other_stratum) - queue.append(other_source) + stratum.add_dependency(other_stratum) + + queue.append(other_source) # 'name' here is the chunk artifact name - chunk_artifacts = [] - processed_artifacts = [] - name_to_processed_artifact = {} + name_to_processed_artifacts = {} - for info in stratum.source.morphology['chunks']: + for info in source.morphology['chunks']: chunk_source = self._source_pool.lookup( info['repo'], info['ref'], '%s.morph' % info['morph']) - possible_names = chunk_source.morphology.builds_artifacts - if not info['name'] in possible_names: - raise UndefinedChunkArtifactError(stratum.source, info['name']) - - chunk_artifact = chunk_source.artifacts[info['name']] - chunk_artifacts.append(chunk_artifact) - - artifacts.append(chunk_artifact) - - stratum.add_dependency(chunk_artifact) - - for other_stratum in strata: - chunk_artifact.add_dependency(other_stratum) + chunk_name = chunk_source.morphology['name'] # Resolve now to avoid a search for the parent morphology later chunk_source.build_mode = info['build-mode'] @@ -215,23 +204,45 @@ class ArtifactResolver(object): build_depends = info.get('build-depends', None) - if build_depends is None: - for earlier_artifact in processed_artifacts: - if earlier_artifact.depends_on(chunk_artifact): - raise MutualDependencyError( - chunk_artifact, earlier_artifact) - chunk_artifact.add_dependency(earlier_artifact) - elif isinstance(build_depends, list): + for ca_name in chunk_source.split_rules.artifacts: + chunk_artifact = chunk_source.artifacts[ca_name] + + # Add our stratum's build depends as dependencies of this chunk + for other_stratum in stratum_build_depends: + chunk_artifact.add_dependency(other_stratum) + + # Add dependencies between chunks mentioned in this stratum + if isinstance(build_depends, list): for name in build_depends: - other_artifact = name_to_processed_artifact.get(name, None) - if other_artifact: - chunk_artifact.add_dependency(other_artifact) - else: + if name not in name_to_processed_artifacts: raise DependencyOrderError( - stratum, info['name'], name) + source, info['name'], name) + other_artifacts = name_to_processed_artifacts[name] + for other_artifact in other_artifacts: + for ca_name in chunk_source.split_rules.artifacts: + chunk_artifact = chunk_source.artifacts[ca_name] + chunk_artifact.add_dependency(other_artifact) else: - raise DependencyFormatError(stratum, info['name']) - processed_artifacts.append(chunk_artifact) - name_to_processed_artifact[info['name']] = chunk_artifact + raise DependencyFormatError(source, info['name']) + + # Add build dependencies between our stratum's artifacts + # and the chunk artifacts produced by this stratum. + matches, overlaps, unmatched = source.split_rules.partition( + ((chunk_name, ca_name) for ca_name + in chunk_source.split_rules.artifacts)) + for stratum in strata: + for (chunk_name, ca_name) in matches[stratum.name]: + chunk_artifact = chunk_source.artifacts[ca_name] + stratum.add_dependency(chunk_artifact) + # Only return chunks required to build strata we need + if chunk_artifact not in artifacts: + artifacts.append(chunk_artifact) + + + # Add these chunks to the processed artifacts, so other + # chunks may refer to them. + name_to_processed_artifacts[info['name']] = \ + [chunk_source.artifacts[n] for n + in chunk_source.split_rules.artifacts] return artifacts diff --git a/morphlib/artifactresolver_tests.py b/morphlib/artifactresolver_tests.py index b685902e..3a9b7f55 100644 --- a/morphlib/artifactresolver_tests.py +++ b/morphlib/artifactresolver_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,6 +14,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import itertools import json import unittest @@ -27,16 +28,15 @@ class FakeChunkMorphology(morphlib.morph2.Morphology): if artifact_names: # fake a list of artifacts - artifacts = {} + artifacts = [] for artifact_name in artifact_names: - artifacts[artifact_name] = [artifact_name] - text = (''' - { - "name": "%s", + artifacts.append({'artifact': artifact_name, + 'include': artifact_name}) + text = json.dumps({ + "name": name, "kind": "chunk", - "chunks": %s - } - ''' % (name, json.dumps(artifacts))) + "products": artifacts + }) self.builds_artifacts = artifact_names else: text = (''' @@ -61,7 +61,8 @@ class FakeStratumMorphology(morphlib.morph2.Morphology): 'name': source_name, 'morph': morph, 'repo': repo, - 'ref': ref + 'ref': ref, + 'build-depends': [], }) build_depends_list = [] for morph, repo, ref in build_depends: @@ -114,33 +115,37 @@ class ArtifactResolverTests(unittest.TestCase): artifacts = self.resolver.resolve_artifacts(pool) - self.assertEqual(len(artifacts), 1) + self.assertEqual(len(artifacts), + sum(len(s.split_rules.artifacts) for s in pool)) - self.assertEqual(artifacts[0].source, source) - self.assertEqual(artifacts[0].name, 'chunk') - self.assertEqual(artifacts[0].dependencies, []) - self.assertEqual(artifacts[0].dependents, []) + for artifact in artifacts: + self.assertEqual(artifact.source, source) + self.assertTrue(artifact.name.startswith('chunk')) + self.assertEqual(artifact.dependencies, []) + self.assertEqual(artifact.dependents, []) - def test_resolve_single_chunk_with_one_artifact(self): + def test_resolve_single_chunk_with_one_new_artifact(self): pool = morphlib.sourcepool.SourcePool() - morph = FakeChunkMorphology('chunk', ['chunk-runtime']) + morph = FakeChunkMorphology('chunk', ['chunk-foobar']) source = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') pool.add(source) artifacts = self.resolver.resolve_artifacts(pool) - self.assertEqual(len(artifacts), 1) - self.assertEqual(artifacts[0].source, source) - self.assertEqual(artifacts[0].name, 'chunk-runtime') - self.assertEqual(artifacts[0].dependencies, []) - self.assertEqual(artifacts[0].dependents, []) + self.assertEqual(len(artifacts), + sum(len(s.split_rules.artifacts) for s in pool)) - def test_resolve_single_chunk_with_two_artifact(self): + foobartifact, = (a for a in artifacts if a.name == 'chunk-foobar') + self.assertEqual(foobartifact.source, source) + self.assertEqual(foobartifact.dependencies, []) + self.assertEqual(foobartifact.dependents, []) + + def test_resolve_single_chunk_with_two_new_artifacts(self): pool = morphlib.sourcepool.SourcePool() - morph = FakeChunkMorphology('chunk', ['chunk-runtime', 'chunk-devel']) + morph = FakeChunkMorphology('chunk', ['chunk-baz', 'chunk-qux']) source = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') pool.add(source) @@ -148,88 +153,16 @@ class ArtifactResolverTests(unittest.TestCase): artifacts = self.resolver.resolve_artifacts(pool) artifacts.sort(key=lambda a: a.name) - self.assertEqual(len(artifacts), 2) - - self.assertEqual(artifacts[0].source, source) - self.assertEqual(artifacts[0].name, 'chunk-devel') - self.assertEqual(artifacts[0].dependencies, []) - self.assertEqual(artifacts[0].dependents, []) - - self.assertEqual(artifacts[1].source, source) - self.assertEqual(artifacts[1].name, 'chunk-runtime') - self.assertEqual(artifacts[1].dependencies, []) - self.assertEqual(artifacts[1].dependents, []) - - def test_resolve_a_single_empty_stratum(self): - pool = morphlib.sourcepool.SourcePool() - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "foo", - "kind": "stratum" - } - ''') - morph.builds_artifacts = ['foo'] - stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'foo.morph') - pool.add(stratum) - - artifacts = self.resolver.resolve_artifacts(pool) - - self.assertEqual(artifacts[0].source, stratum) - self.assertEqual(artifacts[0].name, 'foo') - self.assertEqual(artifacts[0].dependencies, []) - self.assertEqual(artifacts[0].dependents, []) - - def test_resolve_a_single_empty_system(self): - pool = morphlib.sourcepool.SourcePool() - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "foo", - "kind": "system" - } - ''') - morph.builds_artifacts = ['foo-rootfs'] - system = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'foo.morph') - pool.add(system) - - artifacts = self.resolver.resolve_artifacts(pool) - - self.assertEqual(artifacts[0].source, system) - self.assertEqual(artifacts[0].name, 'foo-rootfs') - self.assertEqual(artifacts[0].dependencies, []) - self.assertEqual(artifacts[0].dependents, []) - - def test_resolve_a_single_empty_arm_system(self): - pool = morphlib.sourcepool.SourcePool() - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "foo", - "kind": "system", - "arch": "armv7" - } - ''') - morph.builds_artifacts = ['foo-rootfs', 'foo-kernel'] - system = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'foo.morph') - pool.add(system) - - artifacts = self.resolver.resolve_artifacts(pool) + self.assertEqual(len(artifacts), + sum(len(s.split_rules.artifacts) for s in pool)) - self.assertTrue(any((a.source == system and a.name == 'foo-rootfs' and - a.dependencies == [] and a.dependents == []) - for a in artifacts)) - self.assertTrue(any((a.source == system and a.name == 'foo-kernel' and - a.dependencies == [] and a.dependents == []) - for a in artifacts)) + for name in ('chunk-baz', 'chunk-qux'): + artifact, = (a for a in artifacts if a.name == name) + self.assertEqual(artifact.source, source) + self.assertEqual(artifact.dependencies, []) + self.assertEqual(artifact.dependents, []) - def test_resolve_stratum_and_chunk_with_no_subartifacts(self): + def test_resolve_stratum_and_chunk(self): pool = morphlib.sourcepool.SourcePool() morph = FakeChunkMorphology('chunk') @@ -245,103 +178,36 @@ class ArtifactResolverTests(unittest.TestCase): artifacts = self.resolver.resolve_artifacts(pool) - self.assertEqual(len(artifacts), 2) + self.assertEqual(len(artifacts), + sum(len(s.split_rules.artifacts) for s in pool)) - self.assertEqual(artifacts[0].source, stratum) - self.assertEqual(artifacts[0].name, 'stratum') - self.assertEqual(artifacts[0].dependencies, [artifacts[1]]) - self.assertEqual(artifacts[0].dependents, []) + stratum_artifacts = set(a for a in artifacts if a.source == stratum) + chunk_artifacts = set(a for a in artifacts if a.source == chunk) - self.assertEqual(artifacts[1].source, chunk) - self.assertEqual(artifacts[1].name, 'chunk') - self.assertEqual(artifacts[1].dependencies, []) - self.assertEqual(artifacts[1].dependents, [artifacts[0]]) + for stratum_artifact in stratum_artifacts: + self.assertTrue(stratum_artifact.name.startswith('stratum')) + self.assertEqual(stratum_artifact.dependents, []) + self.assertTrue(any(dep in chunk_artifacts + for dep in stratum_artifact.dependencies)) - def test_resolve_stratum_and_chunk_with_two_subartifacts(self): - pool = morphlib.sourcepool.SourcePool() - - morph = FakeChunkMorphology('chunk', ['chunk-devel', 'chunk-runtime']) - chunk = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') - pool.add(chunk) - - morph = FakeStratumMorphology( - 'stratum', - chunks=[ - ('chunk-devel', 'chunk', 'repo', 'ref'), - ('chunk-runtime', 'chunk', 'repo', 'ref') - ]) - stratum = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'stratum.morph') - pool.add(stratum) - - artifacts = self.resolver.resolve_artifacts(pool) - - self.assertEqual(len(artifacts), 3) - - self.assertEqual(artifacts[0].source, stratum) - self.assertEqual(artifacts[0].name, 'stratum') - self.assertEqual(artifacts[0].dependencies, - [artifacts[1], artifacts[2]]) - self.assertEqual(artifacts[0].dependents, []) - - self.assertEqual(artifacts[1].source, chunk) - self.assertEqual(artifacts[1].name, 'chunk-devel') - self.assertEqual(artifacts[1].dependencies, []) - self.assertEqual(artifacts[1].dependents, [artifacts[0], artifacts[2]]) + for chunk_artifact in chunk_artifacts: + self.assertTrue(chunk_artifact.name.startswith('chunk')) + self.assertEqual(chunk_artifact.dependencies, []) + self.assertTrue(any(dep in stratum_artifacts + for dep in chunk_artifact.dependents)) - self.assertEqual(artifacts[2].source, chunk) - self.assertEqual(artifacts[2].name, 'chunk-runtime') - self.assertEqual(artifacts[2].dependencies, [artifacts[1]]) - self.assertEqual(artifacts[2].dependents, [artifacts[0]]) - - def test_resolve_stratum_and_chunk_with_one_used_subartifacts(self): + def test_resolve_stratum_and_chunk_with_two_new_artifacts(self): pool = morphlib.sourcepool.SourcePool() - morph = FakeChunkMorphology('chunk', ['chunk-devel', 'chunk-runtime']) + morph = FakeChunkMorphology('chunk', ['chunk-foo', 'chunk-bar']) chunk = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') pool.add(chunk) morph = FakeStratumMorphology( 'stratum', - chunks=[('chunk-runtime', 'chunk', 'repo', 'ref')]) - stratum = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'stratum.morph') - pool.add(stratum) - - artifacts = self.resolver.resolve_artifacts(pool) - - self.assertEqual(len(artifacts), 2) - - self.assertEqual(artifacts[0].source, stratum) - self.assertEqual(artifacts[0].name, 'stratum') - self.assertEqual(artifacts[0].dependencies, [artifacts[1]]) - self.assertEqual(artifacts[0].dependents, []) - - self.assertEqual(artifacts[1].source, chunk) - self.assertEqual(artifacts[1].name, 'chunk-runtime') - self.assertEqual(artifacts[1].dependencies, []) - self.assertEqual(artifacts[1].dependents, [artifacts[0]]) - - def test_resolving_two_different_chunk_artifacts_in_a_stratum(self): - pool = morphlib.sourcepool.SourcePool() - - morph = FakeChunkMorphology('foo') - foo_chunk = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'foo.morph') - pool.add(foo_chunk) - - morph = FakeChunkMorphology('bar') - bar_chunk = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'bar.morph') - pool.add(bar_chunk) - - morph = FakeStratumMorphology( - 'stratum', chunks=[ - ('foo', 'foo', 'repo', 'ref'), - ('bar', 'bar', 'repo', 'ref') + ('chunk', 'chunk', 'repo', 'ref'), ]) stratum = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'stratum.morph') @@ -349,114 +215,35 @@ class ArtifactResolverTests(unittest.TestCase): artifacts = self.resolver.resolve_artifacts(pool) - self.assertEqual(len(artifacts), 3) - - self.assertEqual(artifacts[0].source, stratum) - self.assertEqual(artifacts[0].name, 'stratum') - self.assertEqual(artifacts[0].dependencies, - [artifacts[1], artifacts[2]]) - self.assertEqual(artifacts[0].dependents, []) - - self.assertEqual(artifacts[1].source, foo_chunk) - self.assertEqual(artifacts[1].name, 'foo') - self.assertEqual(artifacts[1].dependencies, []) - self.assertEqual(artifacts[1].dependents, [artifacts[0], artifacts[2]]) - - self.assertEqual(artifacts[2].source, bar_chunk) - self.assertEqual(artifacts[2].name, 'bar') - self.assertEqual(artifacts[2].dependencies, [artifacts[1]]) - self.assertEqual(artifacts[2].dependents, [artifacts[0]]) - - def test_resolving_artifacts_for_a_chain_of_two_strata(self): - pool = morphlib.sourcepool.SourcePool() - - morph = FakeStratumMorphology('stratum1') - stratum1 = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'stratum1.morph') - pool.add(stratum1) - - morph = FakeStratumMorphology( - 'stratum2', - chunks=[], - build_depends=[('stratum1', 'repo', 'ref')]) - stratum2 = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'stratum2.morph') - pool.add(stratum2) - - artifacts = self.resolver.resolve_artifacts(pool) + self.assertEqual(len(artifacts), + sum(len(s.split_rules.artifacts) for s in pool)) - self.assertEqual(len(artifacts), 2) + stratum_artifacts = set(a for a in artifacts if a.source == stratum) + chunk_artifacts = set(a for a in artifacts if a.source == chunk) - self.assertEqual(artifacts[0].source, stratum1) - self.assertEqual(artifacts[0].name, 'stratum1') - self.assertEqual(artifacts[0].dependencies, []) - self.assertEqual(artifacts[0].dependents, [artifacts[1]]) + for stratum_artifact in stratum_artifacts: + self.assertTrue(stratum_artifact.name.startswith('stratum')) + self.assertEqual(stratum_artifact.dependents, []) + self.assertTrue(any(dep in chunk_artifacts + for dep in stratum_artifact.dependencies)) - self.assertEqual(artifacts[1].source, stratum2) - self.assertEqual(artifacts[1].name, 'stratum2') - self.assertEqual(artifacts[1].dependencies, [artifacts[0]]) - self.assertEqual(artifacts[1].dependents, []) + for chunk_artifact in chunk_artifacts: + self.assertTrue(chunk_artifact.name.startswith('chunk')) + self.assertEqual(chunk_artifact.dependencies, []) + self.assertTrue(any(dep in stratum_artifacts + for dep in chunk_artifact.dependents)) - def test_resolving_with_a_stratum_and_chunk_dependency_mix(self): + def test_resolving_artifacts_for_a_system_with_two_dependent_strata(self): pool = morphlib.sourcepool.SourcePool() - morph = FakeStratumMorphology('stratum1') - stratum1 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'stratum1.morph') - pool.add(stratum1) - - morph = FakeStratumMorphology( - 'stratum2', - chunks=[ - ('chunk1', 'chunk1', 'repo', 'original/ref'), - ('chunk2', 'chunk2', 'repo', 'original/ref') - ], - build_depends=[('stratum1', 'repo', 'original/ref')]) - stratum2 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'stratum2.morph') - pool.add(stratum2) - morph = FakeChunkMorphology('chunk1') chunk1 = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk1.morph') pool.add(chunk1) - morph = FakeChunkMorphology('chunk2') - chunk2 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk2.morph') - pool.add(chunk2) - - artifacts = self.resolver.resolve_artifacts(pool) - - self.assertEqual(len(artifacts), 4) - - self.assertEqual(artifacts[0].source, stratum1) - self.assertEqual(artifacts[0].name, 'stratum1') - self.assertEqual(artifacts[0].dependencies, []) - self.assertEqual(artifacts[0].dependents, - [artifacts[1], artifacts[2], artifacts[3]]) - - self.assertEqual(artifacts[1].source, stratum2) - self.assertEqual(artifacts[1].name, 'stratum2') - self.assertEqual(artifacts[1].dependencies, - [artifacts[0], artifacts[2], artifacts[3]]) - self.assertEqual(artifacts[1].dependents, []) - - self.assertEqual(artifacts[2].source, chunk1) - self.assertEqual(artifacts[2].name, 'chunk1') - self.assertEqual(artifacts[2].dependencies, [artifacts[0]]) - self.assertEqual(artifacts[2].dependents, [artifacts[1], artifacts[3]]) - - self.assertEqual(artifacts[3].source, chunk2) - self.assertEqual(artifacts[3].name, 'chunk2') - self.assertEqual(artifacts[3].dependencies, - [artifacts[0], artifacts[2]]) - self.assertEqual(artifacts[3].dependents, [artifacts[1]]) - - def test_resolving_artifacts_for_a_system_with_two_strata(self): - pool = morphlib.sourcepool.SourcePool() - - morph = FakeStratumMorphology('stratum1') + morph = FakeStratumMorphology( + 'stratum1', + chunks=[('chunk1', 'chunk1', 'repo', 'original/ref')]) stratum1 = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'stratum1.morph') pool.add(stratum1) @@ -485,31 +272,64 @@ class ArtifactResolverTests(unittest.TestCase): 'repo', 'ref', 'sha1', 'tree', morph, 'system.morph') pool.add(system) + morph = FakeChunkMorphology('chunk2') + chunk2 = morphlib.source.Source( + 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk2.morph') + pool.add(chunk2) + morph = FakeStratumMorphology( - 'stratum2', chunks=[], build_depends=[('stratum1', 'repo', 'ref')]) + 'stratum2', + chunks=[('chunk2', 'chunk2', 'repo', 'original/ref')], + build_depends=[('stratum1', 'repo', 'ref')]) stratum2 = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'stratum2.morph') pool.add(stratum2) artifacts = self.resolver.resolve_artifacts(pool) - self.assertEqual(len(artifacts), 3) - - self.assertEqual(artifacts[0].source, stratum1) - self.assertEqual(artifacts[0].name, 'stratum1') - self.assertEqual(artifacts[0].dependencies, []) - self.assertEqual(artifacts[0].dependents, [artifacts[1], artifacts[2]]) - - self.assertEqual(artifacts[1].source, system) - self.assertEqual(artifacts[1].name, 'system-rootfs') - self.assertEqual(artifacts[1].dependencies, - [artifacts[0], artifacts[2]]) - self.assertEqual(artifacts[1].dependents, []) - - self.assertEqual(artifacts[2].source, stratum2) - self.assertEqual(artifacts[2].name, 'stratum2') - self.assertEqual(artifacts[2].dependencies, [artifacts[0]]) - self.assertEqual(artifacts[2].dependents, [artifacts[1]]) + self.assertEqual(len(artifacts), + sum(len(s.split_rules.artifacts) for s in pool)) + + system_artifacts = set(a for a in artifacts if a.source == system) + stratum1_artifacts = set(a for a in artifacts if a.source == stratum1) + chunk1_artifacts = set(a for a in artifacts if a.source == chunk1) + stratum2_artifacts = set(a for a in artifacts if a.source == stratum2) + chunk2_artifacts = set(a for a in artifacts if a.source == chunk2) + + def assert_depended_on_by_some(artifact, parents): + self.assertNotEqual(len(artifact.dependents), 0) + self.assertTrue(any(a in artifact.dependents for a in parents)) + def assert_depended_on_by_all(artifact, parents): + self.assertNotEqual(len(artifact.dependents), 0) + self.assertTrue(all(a in artifact.dependents for a in parents)) + def assert_depends_on_some(artifact, children): + self.assertNotEqual(len(artifact.dependencies), 0) + self.assertTrue(any(a in children for a in artifact.dependencies)) + def assert_depends_on_all(artifact, children): + self.assertNotEqual(len(artifact.dependencies), 0) + self.assertTrue(all(a in children for a in artifact.dependencies)) + + for c1_a in chunk1_artifacts: + self.assertEqual(c1_a.dependencies, []) + assert_depended_on_by_some(c1_a, stratum1_artifacts) + + for st1_a in stratum1_artifacts: + assert_depends_on_some(st1_a, chunk1_artifacts) + assert_depended_on_by_all(st1_a, chunk2_artifacts) + assert_depended_on_by_some(st1_a, system_artifacts) + + for c2_a in chunk2_artifacts: + assert_depends_on_all(c2_a, stratum1_artifacts) + assert_depended_on_by_some(c2_a, stratum2_artifacts) + + for st2_a in stratum2_artifacts: + assert_depends_on_some(st2_a, chunk2_artifacts) + assert_depended_on_by_some(st2_a, system_artifacts) + + for sy_a in system_artifacts: + self.assertEqual(sy_a.dependents, []) + assert_depends_on_some(sy_a, stratum1_artifacts) + assert_depends_on_some(sy_a, stratum2_artifacts) def test_resolving_stratum_with_explicit_chunk_dependencies(self): pool = morphlib.sourcepool.SourcePool() @@ -566,49 +386,33 @@ class ArtifactResolverTests(unittest.TestCase): artifacts = self.resolver.resolve_artifacts(pool) - self.assertEqual(len(artifacts), 4) - - self.assertEqual(artifacts[0].source, stratum) - self.assertEqual(artifacts[0].name, 'stratum') - self.assertEqual(artifacts[0].dependencies, - [artifacts[1], artifacts[2], artifacts[3]]) - self.assertEqual(artifacts[0].dependents, []) - - self.assertEqual(artifacts[1].source, chunk1) - self.assertEqual(artifacts[1].name, 'chunk1') - self.assertEqual(artifacts[1].dependencies, []) - self.assertEqual(artifacts[1].dependents, - [artifacts[0], artifacts[3]]) - - self.assertEqual(artifacts[2].source, chunk2) - self.assertEqual(artifacts[2].name, 'chunk2') - self.assertEqual(artifacts[2].dependencies, []) - self.assertEqual(artifacts[2].dependents, [artifacts[0], artifacts[3]]) - - self.assertEqual(artifacts[3].source, chunk3) - self.assertEqual(artifacts[3].name, 'chunk3') - self.assertEqual(artifacts[3].dependencies, - [artifacts[1], artifacts[2]]) - self.assertEqual(artifacts[3].dependents, [artifacts[0]]) - - def test_detection_of_invalid_chunk_artifact_references(self): - pool = morphlib.sourcepool.SourcePool() - - morph = FakeChunkMorphology('chunk') - chunk = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') - pool.add(chunk) - - morph = FakeStratumMorphology( - 'stratum', - chunks=[('chunk-runtime', 'chunk', 'repo', 'ref')]) - stratum = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'stratum.morph') - pool.add(stratum) - - self.assertRaises( - morphlib.artifactresolver.UndefinedChunkArtifactError, - self.resolver.resolve_artifacts, pool) + self.assertEqual(len(artifacts), + sum(len(s.split_rules.artifacts) for s in pool)) + + stratum_artifacts = set(a for a in artifacts if a.source == stratum) + chunk_artifacts = [set(a for a in artifacts if a.source == source) + for source in (chunk1, chunk2, chunk3)] + all_chunks = set(itertools.chain.from_iterable(chunk_artifacts)) + + for st_a in stratum_artifacts: + self.assertEqual(st_a.dependents, []) + # This stratum depends on some chunk artifacts + self.assertTrue(any(a in st_a.dependencies for a in all_chunks)) + + for ca in chunk_artifacts[2]: + # There's a stratum dependent on this artifact + self.assertTrue(any(a in stratum_artifacts for a in ca.dependents)) + # chunk3's artifacts depend on chunk1 and chunk2's artifacts + self.assertEqual(set(ca.dependencies), + chunk_artifacts[0] | chunk_artifacts[1]) + + for ca in itertools.chain.from_iterable(chunk_artifacts[0:1]): + self.assertEqual(ca.dependencies, []) + # There's a stratum dependent on this artifact + self.assertTrue(any(a in stratum_artifacts for a in ca.dependents)) + # All chunk3's artifacts depend on this artifact + self.assertTrue(all(c3a in ca.dependents + for c3a in chunk_artifacts[2])) def test_detection_of_mutual_dependency_between_two_strata(self): pool = morphlib.sourcepool.SourcePool() @@ -632,97 +436,6 @@ class ArtifactResolverTests(unittest.TestCase): self.assertRaises(morphlib.artifactresolver.MutualDependencyError, self.resolver.resolve_artifacts, pool) - def test_detection_of_mutual_dependency_between_consecutive_chunks(self): - pool = morphlib.sourcepool.SourcePool() - - morph = FakeStratumMorphology( - 'stratum1', - chunks=[ - ('chunk1', 'chunk1', 'repo', 'original/ref'), - ('chunk2', 'chunk2', 'repo', 'original/ref') - ]) - stratum1 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'stratum1.morph') - pool.add(stratum1) - - morph = FakeStratumMorphology( - 'stratum2', - chunks=[ - ('chunk2', 'chunk2', 'repo', 'original/ref'), - ('chunk1', 'chunk1', 'repo', 'original/ref') - ], - build_depends=[('stratum1', 'repo', 'original/ref')]) - stratum2 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'stratum2.morph') - pool.add(stratum2) - - morph = FakeChunkMorphology('chunk1') - chunk1 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk1.morph') - pool.add(chunk1) - - morph = FakeChunkMorphology('chunk2') - chunk2 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk2.morph') - pool.add(chunk2) - - self.assertRaises(morphlib.artifactresolver.MutualDependencyError, - self.resolver.resolve_artifacts, pool) - - if 0: - # This situation is currently not possible - def test_graceful_handling_of_self_dependencies_of_chunks(self): - pool = morphlib.sourcepool.SourcePool() - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "stratum", - "kind": "stratum", - "chunks": [ - { - "alias": "same-chunk-runtime", - "name": "chunk-runtime", - "morph": "chunk", - "repo": "repo", - "ref": "original/ref" - }, - { - "name": "chunk-runtime", - "morph": "chunk", - "repo": "repo", - "ref": "original/ref", - "build-depends": [ - "same-chunk-runtime" - ] - } - ] - } - ''') - morph.builds_artifacts = ['stratum'] - stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'stratum.morph') - pool.add(stratum) - - morph = FakeChunkMorphology('chunk') - chunk = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk.morph') - pool.add(chunk) - - artifacts = self.resolver.resolve_artifacts(pool) - - self.assertEqual(len(artifacts), 2) - - self.assertEqual(artifacts[0].source, stratum) - self.assertEqual(artifacts[0].name, 'stratum') - self.assertEqual(artifacts[0].dependencies, [artifacts[1]]) - self.assertEqual(artifacts[0].dependents, []) - - self.assertEqual(artifacts[1].source, chunk) - self.assertEqual(artifacts[1].name, 'chunk') - self.assertEqual(artifacts[1].dependencies, []) - self.assertEqual(artifacts[1].dependents, [artifacts[0]]) - def test_detection_of_chunk_dependencies_in_invalid_order(self): pool = morphlib.sourcepool.SourcePool() @@ -796,3 +509,8 @@ class ArtifactResolverTests(unittest.TestCase): self.assertRaises(morphlib.artifactresolver.DependencyFormatError, self.resolver.resolve_artifacts, pool) + + +# TODO: Expand test suite to include better dependency checking, many +# tests were removed due to the fundamental change in how artifacts +# and dependencies are constructed |