diff options
-rwxr-xr-x | morph | 4 | ||||
-rw-r--r-- | morphlib/__init__.py | 1 | ||||
-rw-r--r-- | morphlib/artifact.py | 4 | ||||
-rw-r--r-- | morphlib/artifact_tests.py | 11 | ||||
-rw-r--r-- | morphlib/artifactresolver.py | 29 | ||||
-rw-r--r-- | morphlib/artifactresolver_tests.py | 42 | ||||
-rw-r--r-- | morphlib/buildgraph.py | 208 | ||||
-rw-r--r-- | morphlib/buildgraph_tests.py | 851 | ||||
-rw-r--r-- | morphlib/buildorder_tests.py | 34 | ||||
-rw-r--r-- | morphlib/cachekeycomputer.py | 24 | ||||
-rw-r--r-- | morphlib/cachekeycomputer_tests.py | 35 | ||||
-rw-r--r-- | morphlib/localartifactcache.py | 9 | ||||
-rw-r--r-- | morphlib/localartifactcache_tests.py | 4 |
13 files changed, 76 insertions, 1180 deletions
@@ -259,9 +259,7 @@ class Morph(cliapp.Application): local_repo_cache, remote_repo_cache, repo, ref, filename) - env = morphlib.buildenvironment.BuildEnvironment(self.settings) - computer = morphlib.cachekeycomputer.CacheKeyComputer(env) - resolver = morphlib.artifactresolver.ArtifactResolver(computer) + resolver = morphlib.artifactresolver.ArtifactResolver() artifacts = resolver.resolve_artifacts(pool) self.output.write('dependency graph for %s|%s|%s:\n' % diff --git a/morphlib/__init__.py b/morphlib/__init__.py index be3d7c5c..44b6ce53 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -23,7 +23,6 @@ import bins import blobs import buildcontroller import builddependencygraph -import buildgraph import buildenvironment import buildorder import buildsystem diff --git a/morphlib/artifact.py b/morphlib/artifact.py index f5c2e8a5..3a438747 100644 --- a/morphlib/artifact.py +++ b/morphlib/artifact.py @@ -31,10 +31,10 @@ class Artifact(object): ''' - def __init__(self, source, name, cache_key): + def __init__(self, source, name): self.source = source self.name = name - self.cache_key = cache_key + self.cache_key = None self.dependencies = [] self.dependents = [] diff --git a/morphlib/artifact_tests.py b/morphlib/artifact_tests.py index faeef879..5c912cd7 100644 --- a/morphlib/artifact_tests.py +++ b/morphlib/artifact_tests.py @@ -27,7 +27,7 @@ class ArtifactTests(unittest.TestCase): { "chunk": "chunk", "kind": "chunk", - "artifacts": { + "chunks": { "chunk-runtime": [ "usr/bin", "usr/sbin", @@ -42,12 +42,11 @@ class ArtifactTests(unittest.TestCase): ''') self.source = morphlib.source.Source( 'repo', 'ref', 'sha1', morph, 'chunk.morph') - self.cache_key = 'cachekey' self.artifact_name = 'chunk-runtime' self.artifact = morphlib.artifact.Artifact( - self.source, self.artifact_name, self.cache_key) + self.source, self.artifact_name) self.other = morphlib.artifact.Artifact( - self.source, self.artifact_name, self.cache_key) + self.source, self.artifact_name) def test_constructor_sets_source(self): self.assertEqual(self.artifact.source, self.source) @@ -55,8 +54,8 @@ class ArtifactTests(unittest.TestCase): def test_constructor_sets_name(self): self.assertEqual(self.artifact.name, self.artifact_name) - def test_constructor_sets_cache_key(self): - self.assertEqual(self.artifact.cache_key, self.cache_key) + def test_constructor_initializes_cache_key_as_none(self): + self.assertEqual(self.artifact.cache_key, None) def test_sets_dependencies_to_empty(self): self.assertEqual(self.artifact.dependencies, []) diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index df3fdf87..25836b41 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -70,8 +70,7 @@ class ArtifactResolver(object): ''' - def __init__(self, cache_key_computer): - self.cache_key_computer = cache_key_computer + def __init__(self): self._cached_artifacts = None self._added_artifacts = None self._source_pool = None @@ -93,11 +92,9 @@ class ArtifactResolver(object): while queue: source = queue.popleft() - cache_key = self.cache_key_computer.compute_key(source) - if source.morphology['kind'] == 'system': artifact = self._get_artifact( - source, source.morphology['name'], cache_key) + source, source.morphology['name']) if not artifact in self._added_artifacts: artifacts.append(artifact) @@ -112,7 +109,7 @@ class ArtifactResolver(object): self._added_artifacts.add(artifact) elif source.morphology['kind'] == 'stratum': artifact = self._get_artifact( - source, source.morphology['name'], cache_key) + source, source.morphology['name']) if not artifact in self._added_artifacts: artifacts.append(artifact) @@ -128,7 +125,7 @@ class ArtifactResolver(object): elif source.morphology['kind'] == 'chunk': names = self._chunk_artifact_names(source) for name in names: - artifact = self._get_artifact(source, name, cache_key) + artifact = self._get_artifact(source, name) if not artifact in self._added_artifacts: artifacts.append(artifact) self._added_artifacts.add(artifact) @@ -143,12 +140,12 @@ class ArtifactResolver(object): if x.morphology['kind'] != 'chunk'] return collections.deque(sources) - def _get_artifact(self, source, name, cache_key): - info = (source, name, cache_key) + def _get_artifact(self, source, name): + info = (source, name) if info in self._cached_artifacts: return self._cached_artifacts[info] else: - artifact = morphlib.artifact.Artifact(info[0], info[1], info[2]) + artifact = morphlib.artifact.Artifact(info[0], info[1]) self._cached_artifacts[info] = artifact return artifact @@ -161,8 +158,7 @@ class ArtifactResolver(object): system.source.original_ref, '%s.morph' % stratum_name) - cache_key = self.cache_key_computer.compute_key(source) - stratum = self._get_artifact(source, stratum_name, cache_key) + stratum = self._get_artifact(source, stratum_name) system.add_dependency(stratum) queue.append(source) @@ -183,9 +179,7 @@ class ArtifactResolver(object): stratum.source.original_ref, '%s.morph' % stratum_name) - cache_key = self.cache_key_computer.compute_key(other_source) - other_stratum = self._get_artifact( - other_source, stratum_name, cache_key) + other_stratum = self._get_artifact(other_source, stratum_name) strata.append(other_stratum) @@ -211,10 +205,7 @@ class ArtifactResolver(object): if not info['name'] in possible_names: raise UndefinedChunkArtifactError(stratum.source, info['name']) - cache_key = self.cache_key_computer.compute_key(chunk_source) - - chunk_artifact = self._get_artifact( - chunk_source, info['name'], cache_key) + chunk_artifact = self._get_artifact(chunk_source, info['name']) chunk_artifacts.append(chunk_artifact) artifacts.append(chunk_artifact) diff --git a/morphlib/artifactresolver_tests.py b/morphlib/artifactresolver_tests.py index d77ff5bb..467c882d 100644 --- a/morphlib/artifactresolver_tests.py +++ b/morphlib/artifactresolver_tests.py @@ -20,13 +20,6 @@ import unittest import morphlib -class FakeCacheKeyComputer(object): - '''Fake computer that uses the uppercase source name as the cache key.''' - - def compute_key(self, source): - return source.morphology['name'].upper() - - class FakeChunkMorphology(morphlib.morph2.Morphology): def __init__(self, name, artifact_names=[]): @@ -94,9 +87,7 @@ class FakeStratumMorphology(morphlib.morph2.Morphology): class ArtifactResolverTests(unittest.TestCase): def setUp(self): - self.cache_key_computer = FakeCacheKeyComputer() - self.resolver = morphlib.artifactresolver.ArtifactResolver( - self.cache_key_computer) + self.resolver = morphlib.artifactresolver.ArtifactResolver() def test_resolve_artifacts_using_an_empty_pool(self): pool = morphlib.sourcepool.SourcePool() @@ -117,7 +108,6 @@ class ArtifactResolverTests(unittest.TestCase): self.assertEqual(artifacts[0].source, source) self.assertEqual(artifacts[0].name, 'chunk') - self.assertEqual(artifacts[0].cache_key, 'CHUNK') self.assertEqual(artifacts[0].dependencies, []) self.assertEqual(artifacts[0].dependents, []) @@ -134,7 +124,6 @@ class ArtifactResolverTests(unittest.TestCase): self.assertEqual(len(artifacts), 1) self.assertEqual(artifacts[0].source, source) self.assertEqual(artifacts[0].name, 'chunk-runtime') - self.assertEqual(artifacts[0].cache_key, 'CHUNK') self.assertEqual(artifacts[0].dependencies, []) self.assertEqual(artifacts[0].dependents, []) @@ -152,13 +141,11 @@ class ArtifactResolverTests(unittest.TestCase): self.assertEqual(artifacts[0].source, source) self.assertEqual(artifacts[0].name, 'chunk-devel') - self.assertEqual(artifacts[0].cache_key, 'CHUNK') self.assertEqual(artifacts[0].dependencies, []) self.assertEqual(artifacts[0].dependents, []) self.assertEqual(artifacts[1].source, source) self.assertEqual(artifacts[1].name, 'chunk-runtime') - self.assertEqual(artifacts[1].cache_key, 'CHUNK') self.assertEqual(artifacts[1].dependencies, []) self.assertEqual(artifacts[1].dependents, []) @@ -180,7 +167,6 @@ class ArtifactResolverTests(unittest.TestCase): self.assertEqual(artifacts[0].source, stratum) self.assertEqual(artifacts[0].name, 'foo') - self.assertEqual(artifacts[0].cache_key, 'FOO') self.assertEqual(artifacts[0].dependencies, []) self.assertEqual(artifacts[0].dependents, []) @@ -202,7 +188,6 @@ class ArtifactResolverTests(unittest.TestCase): self.assertEqual(artifacts[0].source, system) self.assertEqual(artifacts[0].name, 'foo') - self.assertEqual(artifacts[0].cache_key, 'FOO') self.assertEqual(artifacts[0].dependencies, []) self.assertEqual(artifacts[0].dependents, []) @@ -226,13 +211,11 @@ class ArtifactResolverTests(unittest.TestCase): self.assertEqual(artifacts[0].source, stratum) self.assertEqual(artifacts[0].name, 'stratum') - self.assertEqual(artifacts[0].cache_key, 'STRATUM') self.assertEqual(artifacts[0].dependencies, [artifacts[1]]) self.assertEqual(artifacts[0].dependents, []) self.assertEqual(artifacts[1].source, chunk) self.assertEqual(artifacts[1].name, 'chunk') - self.assertEqual(artifacts[1].cache_key, 'CHUNK') self.assertEqual(artifacts[1].dependencies, []) self.assertEqual(artifacts[1].dependents, [artifacts[0]]) @@ -259,20 +242,17 @@ class ArtifactResolverTests(unittest.TestCase): self.assertEqual(artifacts[0].source, stratum) self.assertEqual(artifacts[0].name, 'stratum') - self.assertEqual(artifacts[0].cache_key, 'STRATUM') self.assertEqual(artifacts[0].dependencies, [artifacts[1], artifacts[2]]) self.assertEqual(artifacts[0].dependents, []) self.assertEqual(artifacts[1].source, chunk) self.assertEqual(artifacts[1].name, 'chunk-devel') - self.assertEqual(artifacts[1].cache_key, 'CHUNK') self.assertEqual(artifacts[1].dependencies, []) self.assertEqual(artifacts[1].dependents, [artifacts[0], artifacts[2]]) self.assertEqual(artifacts[2].source, chunk) self.assertEqual(artifacts[2].name, 'chunk-runtime') - self.assertEqual(artifacts[2].cache_key, 'CHUNK') self.assertEqual(artifacts[2].dependencies, [artifacts[1]]) self.assertEqual(artifacts[2].dependents, [artifacts[0]]) @@ -298,13 +278,11 @@ class ArtifactResolverTests(unittest.TestCase): self.assertEqual(artifacts[0].source, stratum) self.assertEqual(artifacts[0].name, 'stratum') - self.assertEqual(artifacts[0].cache_key, 'STRATUM') self.assertEqual(artifacts[0].dependencies, [artifacts[1]]) self.assertEqual(artifacts[0].dependents, []) self.assertEqual(artifacts[1].source, chunk) self.assertEqual(artifacts[1].name, 'chunk-runtime') - self.assertEqual(artifacts[1].cache_key, 'CHUNK') self.assertEqual(artifacts[1].dependencies, []) self.assertEqual(artifacts[1].dependents, [artifacts[0]]) @@ -336,20 +314,17 @@ class ArtifactResolverTests(unittest.TestCase): self.assertEqual(artifacts[0].source, stratum) self.assertEqual(artifacts[0].name, 'stratum') - self.assertEqual(artifacts[0].cache_key, 'STRATUM') self.assertEqual(artifacts[0].dependencies, [artifacts[1], artifacts[2]]) self.assertEqual(artifacts[0].dependents, []) self.assertEqual(artifacts[1].source, foo_chunk) self.assertEqual(artifacts[1].name, 'foo') - self.assertEqual(artifacts[1].cache_key, 'FOO') self.assertEqual(artifacts[1].dependencies, []) self.assertEqual(artifacts[1].dependents, [artifacts[0], artifacts[2]]) self.assertEqual(artifacts[2].source, bar_chunk) self.assertEqual(artifacts[2].name, 'bar') - self.assertEqual(artifacts[2].cache_key, 'BAR') self.assertEqual(artifacts[2].dependencies, [artifacts[1]]) self.assertEqual(artifacts[2].dependents, [artifacts[0]]) @@ -372,13 +347,11 @@ class ArtifactResolverTests(unittest.TestCase): self.assertEqual(artifacts[0].source, stratum1) self.assertEqual(artifacts[0].name, 'stratum1') - self.assertEqual(artifacts[0].cache_key, 'STRATUM1') self.assertEqual(artifacts[0].dependencies, []) self.assertEqual(artifacts[0].dependents, [artifacts[1]]) self.assertEqual(artifacts[1].source, stratum2) self.assertEqual(artifacts[1].name, 'stratum2') - self.assertEqual(artifacts[1].cache_key, 'STRATUM2') self.assertEqual(artifacts[1].dependencies, [artifacts[0]]) self.assertEqual(artifacts[1].dependents, []) @@ -415,27 +388,23 @@ class ArtifactResolverTests(unittest.TestCase): self.assertEqual(artifacts[0].source, stratum1) self.assertEqual(artifacts[0].name, 'stratum1') - self.assertEqual(artifacts[0].cache_key, 'STRATUM1') self.assertEqual(artifacts[0].dependencies, []) self.assertEqual(artifacts[0].dependents, [artifacts[1], artifacts[2], artifacts[3]]) self.assertEqual(artifacts[1].source, stratum2) self.assertEqual(artifacts[1].name, 'stratum2') - self.assertEqual(artifacts[1].cache_key, 'STRATUM2') self.assertEqual(artifacts[1].dependencies, [artifacts[0], artifacts[2], artifacts[3]]) self.assertEqual(artifacts[1].dependents, []) self.assertEqual(artifacts[2].source, chunk1) self.assertEqual(artifacts[2].name, 'chunk1') - self.assertEqual(artifacts[2].cache_key, 'CHUNK1') self.assertEqual(artifacts[2].dependencies, [artifacts[0]]) self.assertEqual(artifacts[2].dependents, [artifacts[1], artifacts[3]]) self.assertEqual(artifacts[3].source, chunk2) self.assertEqual(artifacts[3].name, 'chunk2') - self.assertEqual(artifacts[3].cache_key, 'CHUNK2') self.assertEqual(artifacts[3].dependencies, [artifacts[0], artifacts[2]]) self.assertEqual(artifacts[3].dependents, [artifacts[1]]) @@ -474,20 +443,17 @@ class ArtifactResolverTests(unittest.TestCase): self.assertEqual(artifacts[0].source, stratum1) self.assertEqual(artifacts[0].name, 'stratum1') - self.assertEqual(artifacts[0].cache_key, 'STRATUM1') self.assertEqual(artifacts[0].dependencies, []) self.assertEqual(artifacts[0].dependents, [artifacts[1], artifacts[2]]) self.assertEqual(artifacts[1].source, system) self.assertEqual(artifacts[1].name, 'system') - self.assertEqual(artifacts[1].cache_key, 'SYSTEM') self.assertEqual(artifacts[1].dependencies, [artifacts[0], artifacts[2]]) self.assertEqual(artifacts[1].dependents, []) self.assertEqual(artifacts[2].source, stratum2) self.assertEqual(artifacts[2].name, 'stratum2') - self.assertEqual(artifacts[2].cache_key, 'STRATUM2') self.assertEqual(artifacts[2].dependencies, [artifacts[0]]) self.assertEqual(artifacts[2].dependents, [artifacts[1]]) @@ -549,27 +515,23 @@ class ArtifactResolverTests(unittest.TestCase): self.assertEqual(artifacts[0].source, stratum) self.assertEqual(artifacts[0].name, 'stratum') - self.assertEqual(artifacts[0].cache_key, 'STRATUM') self.assertEqual(artifacts[0].dependencies, [artifacts[1], artifacts[2], artifacts[3]]) self.assertEqual(artifacts[0].dependents, []) self.assertEqual(artifacts[1].source, chunk1) self.assertEqual(artifacts[1].name, 'chunk1') - self.assertEqual(artifacts[1].cache_key, 'CHUNK1') self.assertEqual(artifacts[1].dependencies, []) self.assertEqual(artifacts[1].dependents, [artifacts[0], artifacts[3]]) self.assertEqual(artifacts[2].source, chunk2) self.assertEqual(artifacts[2].name, 'chunk2') - self.assertEqual(artifacts[2].cache_key, 'CHUNK2') self.assertEqual(artifacts[2].dependencies, []) self.assertEqual(artifacts[2].dependents, [artifacts[0], artifacts[3]]) self.assertEqual(artifacts[3].source, chunk3) self.assertEqual(artifacts[3].name, 'chunk3') - self.assertEqual(artifacts[3].cache_key, 'CHUNK3') self.assertEqual(artifacts[3].dependencies, [artifacts[1], artifacts[2]]) self.assertEqual(artifacts[3].dependents, [artifacts[0]]) @@ -689,13 +651,11 @@ class ArtifactResolverTests(unittest.TestCase): self.assertEqual(artifacts[0].source, stratum) self.assertEqual(artifacts[0].name, 'stratum') - self.assertEqual(artifacts[0].cache_key, 'STRATUM') self.assertEqual(artifacts[0].dependencies, [artifacts[1]]) self.assertEqual(artifacts[0].dependents, []) self.assertEqual(artifacts[1].source, chunk) self.assertEqual(artifacts[1].name, 'chunk') - self.assertEqual(artifacts[1].cache_key, 'CHUNK') self.assertEqual(artifacts[1].dependencies, []) self.assertEqual(artifacts[1].dependents, [artifacts[0]]) diff --git a/morphlib/buildgraph.py b/morphlib/buildgraph.py deleted file mode 100644 index 687dbb13..00000000 --- a/morphlib/buildgraph.py +++ /dev/null @@ -1,208 +0,0 @@ -# Copyright (C) 2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import cliapp -import collections - - -class MutualDependencyError(cliapp.AppException): - - def __init__(self, a, b): - cliapp.AppException.__init__( - self, 'Cyclic dependency between %s and %s detected' % (a, b)) - - -class CyclicDependencyChainError(cliapp.AppException): - - def __init__(self): - cliapp.AppException.__init__( - self, 'Cyclic dependency detected somewhere') - - -class DependencyOrderError(cliapp.AppException): - - def __init__(self, stratum, chunk, dependency_name): - cliapp.AppException.__init__( - self, 'In stratum %s, chunk %s references its dependency %s ' - 'before it is defined' % (stratum, chunk, dependency_name)) - - -class DependencyFormatError(cliapp.AppException): - - def __init__(self, stratum, chunk): - cliapp.AppException.__init__( - self, 'In stratum %s, chunk %s uses an invalid ' - 'build-depends format' % (stratum, chunk)) - - -class BuildGraph(object): - - def compute_build_order(self, source_pool): - self._realise_dependencies(source_pool) - sorting = self._compute_topological_sorting(source_pool) - groups = self._create_build_groups(sorting) - return groups - - def _realise_dependencies(self, source_pool): - queue = collections.deque(source_pool) - while queue: - source = queue.popleft() - - if source.morphology['kind'] == 'system': - self._realise_system_dependencies(source, queue, source_pool) - elif source.morphology['kind'] == 'stratum': - self._realise_stratum_dependencies(source, queue, source_pool) - - def _realise_system_dependencies(self, system, queue, source_pool): - for stratum_name in system.morphology['strata']: - stratum = source_pool.lookup( - system.repo_name, - system.original_ref, - '%s.morph' % stratum_name) - - system.add_dependency(stratum) - queue.append(stratum) - - def _realise_stratum_dependencies(self, stratum, queue, source_pool): - strata = [] - - if stratum.morphology['build-depends']: - for stratum_name in stratum.morphology['build-depends']: - other_stratum = source_pool.lookup( - stratum.repo_name, - stratum.original_ref, - '%s.morph' % stratum_name) - strata.append(other_stratum) - - if other_stratum.depends_on(stratum): - raise MutualDependencyError(stratum, other_stratum) - - stratum.add_dependency(other_stratum) - queue.append(other_stratum) - - chunks = [] - processed_chunks = [] - name_to_processed_chunk = {} - - for info in stratum.morphology['sources']: - chunk = source_pool.lookup( - info['repo'], - info['ref'], - '%s.morph' % info['morph']) - chunks.append(chunk) - - stratum.add_dependency(chunk) - - for other_stratum in strata: - chunk.add_dependency(other_stratum) - - build_depends = info.get('build-depends', None) - - if build_depends is None: - for earlier_chunk in processed_chunks: - if earlier_chunk.depends_on(chunk): - raise MutualDependencyError(chunk, earlier_chunk) - chunk.add_dependency(earlier_chunk) - elif isinstance(build_depends, list): - for name in build_depends: - other_chunk = name_to_processed_chunk.get(name, None) - if other_chunk: - chunk.add_dependency(other_chunk) - else: - raise DependencyOrderError(stratum, info['name'], name) - else: - raise DependencyFormatError(stratum, info['name']) - processed_chunks.append(chunk) - name_to_processed_chunk[info['name']] = chunk - - def _compute_topological_sorting(self, source_pool): - '''Computes a topological sorting of the build graph. - - A topological sorting basically is the result of a series of - breadth-first searches starting at each leaf node (sources with no - dependencies). Sources are added to the sorting as soon as all their - dependencies have been added (which means that by then, all - dependencies are satisfied). - - For more information, see - http://en.wikipedia.org/wiki/Topological_sorting. - - ''' - - # map sources to sets of satisfied dependencies. this is to detect when - # we can actually add sources to the BFS queue. rather than dropping - # links between nodes, like most topological sorting algorithms do, - # we simply remember all satisfied dependencies and check if all - # of them are met repeatedly - satisfied_dependencies = {} - - # create an empty sorting - sorting = collections.deque() - - # create a set of leafs to start the DFS from - leafs = collections.deque() - for source in source_pool: - satisfied_dependencies[source] = set() - if len(source.dependencies) == 0: - leafs.append(source) - - while len(leafs) > 0: - # fetch a leaf source from the DFS queue - source = leafs.popleft() - - # add it to the sorting - sorting.append(source) - - # mark this dependency as resolved - for dependent in source.dependents: - satisfied_dependencies[dependent].add(source) - - # add the dependent blob as a leaf if all - # its dependencies have been resolved - has = len(satisfied_dependencies[dependent]) - needs = len(dependent.dependencies) - if has == needs: - leafs.append(dependent) - - # if not all dependencies were resolved on the way, we - # have found at least one cyclic dependency - if len(sorting) < len(source_pool): - raise CyclicDependencyChainError() - - return sorting - - def _create_build_groups(self, sorting): - groups = collections.deque() - - if sorting: - # create the first group - group = [] - groups.append(group) - - # traverse the build graph in topological order - for source in sorting: - # add the current item to the current group, or a new group - # if one of its dependencies is in the current one - create_group = False - for dependency in source.dependencies: - if dependency in group: - create_group = True - if create_group: - group = [] - groups.append(group) - group.append(source) - - return groups diff --git a/morphlib/buildgraph_tests.py b/morphlib/buildgraph_tests.py deleted file mode 100644 index 18abcb6c..00000000 --- a/morphlib/buildgraph_tests.py +++ /dev/null @@ -1,851 +0,0 @@ -# Copyright (C) 2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import collections -import unittest - -import morphlib.buildgraph -import morphlib.source - - -class BuildGraphTests(unittest.TestCase): - - def setUp(self): - self.graph = morphlib.buildgraph.BuildGraph() - - def test_create_empty_build_order_for_empty_pool(self): - pool = morphlib.sourcepool.SourcePool() - order = self.graph.compute_build_order(pool) - self.assertEqual(order, collections.deque()) - - def test_build_order_with_a_single_chunk(self): - pool = morphlib.sourcepool.SourcePool() - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "foo", - "kind": "chunk", - "artifacts": { - "foo-runtime": [ "usr/bin" ], - "foo-devel": [ "usr/lib" ] - } - } - ''') - source = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'foo.morph') - pool.add(source) - - order = self.graph.compute_build_order(pool) - desired_order = collections.deque([ - [source] - ]) - self.assertEqual(order, desired_order) - - def test_build_order_with_a_single_empty_stratum(self): - pool = morphlib.sourcepool.SourcePool() - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "foo", - "kind": "stratum" - } - ''') - source = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'foo.morph') - pool.add(source) - - order = self.graph.compute_build_order(pool) - desired_order = collections.deque([ - [source] - ]) - self.assertEqual(order, desired_order) - - def test_build_order_with_a_single_empty_system(self): - pool = morphlib.sourcepool.SourcePool() - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "foo", - "kind": "system" - } - ''') - source = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'foo.morph') - pool.add(source) - - order = self.graph.compute_build_order(pool) - desired_order = collections.deque([ - [source] - ]) - self.assertEqual(order, desired_order) - - def test_build_order_with_a_one_chunk_stratum(self): - pool = morphlib.sourcepool.SourcePool() - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "stratum", - "kind": "stratum", - "sources": [ - { - "name": "chunk", - "repo": "repo", - "ref": "original/ref" - } - ] - } - ''') - stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'stratum.morph') - pool.add(stratum) - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "chunk", - "kind": "chunk", - "artifacts": { - "foo-runtime": [ "usr/bin" ], - "foo-devel": [ "usr/lib" ] - } - } - ''') - chunk = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'chunk.morph') - pool.add(chunk) - - order = self.graph.compute_build_order(pool) - desired_order = collections.deque([ - [chunk], - [stratum] - ]) - self.assertEqual(order, desired_order) - - self.assertEqual(stratum.dependencies, [chunk]) - - def test_build_order_with_a_one_chunk_artifact_stratum(self): - pool = morphlib.sourcepool.SourcePool() - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "stratum", - "kind": "stratum", - "sources": [ - { - "name": "chunk-runtime", - "morph": "chunk", - "repo": "repo", - "ref": "original/ref" - } - ] - } - ''') - stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'stratum.morph') - pool.add(stratum) - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "chunk", - "kind": "chunk", - "artifacts": { - "foo-runtime": [ "usr/bin" ], - "foo-devel": [ "usr/lib" ] - } - } - ''') - chunk = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'chunk.morph') - pool.add(chunk) - - order = self.graph.compute_build_order(pool) - desired_order = collections.deque([ - [chunk], - [stratum] - ]) - self.assertEqual(order, desired_order) - - self.assertEqual(stratum.dependencies, [chunk]) - - def test_build_order_with_stratum_and_implicit_dependencies(self): - pool = morphlib.sourcepool.SourcePool() - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "stratum", - "kind": "stratum", - "sources": [ - { - "name": "first-chunk", - "repo": "repo", - "ref": "original/ref" - }, - { - "name": "second-chunk", - "repo": "repo", - "ref": "original/ref" - }, - { - "name": "third-chunk", - "repo": "repo", - "ref": "original/ref" - } - ] - } - ''') - stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'stratum.morph') - pool.add(stratum) - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "first-chunk", - "kind": "chunk" - } - ''') - first_chunk = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'first-chunk.morph') - pool.add(first_chunk) - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "second-chunk", - "kind": "chunk" - } - ''') - second_chunk = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'second-chunk.morph') - pool.add(second_chunk) - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "third-chunk", - "kind": "chunk" - } - ''') - third_chunk = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'third-chunk.morph') - pool.add(third_chunk) - - order = self.graph.compute_build_order(pool) - desired_order = collections.deque([ - [first_chunk], - [second_chunk], - [third_chunk], - [stratum] - ]) - self.assertEqual(order, desired_order) - - self.assertEqual(first_chunk.dependencies, []) - self.assertEqual(second_chunk.dependencies, [first_chunk]) - self.assertEqual(third_chunk.dependencies, [first_chunk, second_chunk]) - self.assertEqual(stratum.dependencies, - [first_chunk, second_chunk, third_chunk]) - - def test_build_order_with_explicit_dependencies(self): - pool = morphlib.sourcepool.SourcePool() - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "stratum", - "kind": "stratum", - "sources": [ - { - "name": "first-chunk", - "repo": "repo", - "ref": "original/ref", - "build-depends": [] - }, - { - "name": "second-chunk", - "repo": "repo", - "ref": "original/ref", - "build-depends": [] - }, - { - "name": "third-chunk", - "repo": "repo", - "ref": "original/ref", - "build-depends": [ - "first-chunk", - "second-chunk" - ] - } - ] - } - ''') - stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'stratum.morph') - pool.add(stratum) - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "first-chunk", - "kind": "chunk" - } - ''') - first_chunk = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'first-chunk.morph') - pool.add(first_chunk) - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "second-chunk", - "kind": "chunk" - } - ''') - second_chunk = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'second-chunk.morph') - pool.add(second_chunk) - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "third-chunk", - "kind": "chunk" - } - ''') - third_chunk = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'third-chunk.morph') - pool.add(third_chunk) - - order = self.graph.compute_build_order(pool) - desired_order = collections.deque([ - [first_chunk, second_chunk], - [third_chunk], - [stratum] - ]) - self.assertEqual(order, desired_order) - - self.assertEqual(first_chunk.dependencies, []) - self.assertEqual(second_chunk.dependencies, []) - self.assertEqual(third_chunk.dependencies, [first_chunk, second_chunk]) - self.assertEqual(stratum.dependencies, - [first_chunk, second_chunk, third_chunk]) - - def test_build_order_with_stratum_dependencies(self): - pool = morphlib.sourcepool.SourcePool() - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "first-stratum", - "kind": "stratum" - } - ''') - first_stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, - 'first-stratum.morph') - pool.add(first_stratum) - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "second-stratum", - "kind": "stratum", - "build-depends": [ - "first-stratum" - ] - } - ''') - second_stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, - 'second-stratum.morph') - pool.add(second_stratum) - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "third-stratum", - "kind": "stratum", - "build-depends": [ - "second-stratum" - ] - } - ''') - third_stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, - 'third-stratum.morph') - pool.add(third_stratum) - - order = self.graph.compute_build_order(pool) - desired_order = collections.deque([ - [first_stratum], - [second_stratum], - [third_stratum] - ]) - self.assertEqual(order, desired_order) - - self.assertEqual(first_stratum.dependencies, []) - self.assertEqual(second_stratum.dependencies, [first_stratum]) - self.assertEqual(third_stratum.dependencies, [second_stratum]) - - def test_build_order_with_stratum_and_chunk_dependencies(self): - pool = morphlib.sourcepool.SourcePool() - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "first-stratum", - "kind": "stratum" - } - ''') - first_stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, - 'first-stratum.morph') - pool.add(first_stratum) - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "second-stratum", - "kind": "stratum", - "build-depends": [ - "first-stratum" - ], - "sources": [ - { - "name": "first-chunk", - "repo": "repo", - "ref": "original/ref" - }, - { - "name": "second-chunk", - "repo": "repo", - "ref": "original/ref" - } - ] - } - ''') - second_stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, - 'second-stratum.morph') - pool.add(second_stratum) - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "first-chunk", - "kind": "chunk" - } - ''') - first_chunk = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'first-chunk.morph') - pool.add(first_chunk) - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "second-chunk", - "kind": "chunk" - } - ''') - second_chunk = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'second-chunk.morph') - pool.add(second_chunk) - - order = self.graph.compute_build_order(pool) - desired_order = collections.deque([ - [first_stratum], - [first_chunk], - [second_chunk], - [second_stratum] - ]) - self.assertEqual(order, desired_order) - - self.assertEqual(first_stratum.dependencies, []) - self.assertEqual(first_chunk.dependencies, [first_stratum]) - self.assertEqual(second_chunk.dependencies, - [first_stratum, first_chunk]) - self.assertEqual(second_stratum.dependencies, - [first_stratum, first_chunk, second_chunk]) - - def test_build_order_with_a_system_and_two_strata(self): - pool = morphlib.sourcepool.SourcePool() - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "first-stratum", - "kind": "stratum" - } - ''') - first_stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, - 'first-stratum.morph') - pool.add(first_stratum) - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "second-stratum", - "kind": "stratum" - } - ''') - second_stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, - 'second-stratum.morph') - pool.add(second_stratum) - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "system", - "kind": "system", - "strata": [ - "first-stratum", - "second-stratum" - ] - } - ''') - system = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'system.morph') - pool.add(system) - - order = self.graph.compute_build_order(pool) - desired_order = collections.deque([ - [first_stratum, second_stratum], - [system] - ]) - self.assertEqual(order, desired_order) - - self.assertEqual(first_stratum.dependencies, []) - self.assertEqual(second_stratum.dependencies, []) - self.assertEqual(system.dependencies, [first_stratum, second_stratum]) - - def test_detection_of_mutual_dependency_between_two_strata(self): - pool = morphlib.sourcepool.SourcePool() - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "first-stratum", - "kind": "stratum", - "build-depends": [ - "second-stratum" - ] - } - ''') - first_stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, - 'first-stratum.morph') - pool.add(first_stratum) - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "second-stratum", - "kind": "stratum", - "build-depends": [ - "first-stratum" - ] - } - ''') - second_stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, - 'second-stratum.morph') - pool.add(second_stratum) - - self.assertRaises(morphlib.buildgraph.MutualDependencyError, - self.graph.compute_build_order, pool) - - def test_detection_of_mutual_dependency_between_consecutive_chunks(self): - pool = morphlib.sourcepool.SourcePool() - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "first-stratum", - "kind": "stratum", - "sources": [ - { - "name": "first-chunk", - "repo": "repo", - "ref": "original/ref" - }, - { - "name": "second-chunk", - "repo": "repo", - "ref": "original/ref" - } - ] - } - ''') - first_stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, - 'first-stratum.morph') - pool.add(first_stratum) - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "second-stratum", - "kind": "stratum", - "build-depends": [ - "first-stratum" - ], - "sources": [ - { - "name": "second-chunk", - "repo": "repo", - "ref": "original/ref" - }, - { - "name": "first-chunk", - "repo": "repo", - "ref": "original/ref" - } - ] - } - ''') - second_stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, - 'second-stratum.morph') - pool.add(second_stratum) - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "first-chunk", - "kind": "chunk" - } - ''') - first_chunk = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, - 'first-chunk.morph') - pool.add(first_chunk) - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "second-chunk", - "kind": "chunk" - } - ''') - second_chunk = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, - 'second-chunk.morph') - pool.add(second_chunk) - - self.assertRaises(morphlib.buildgraph.MutualDependencyError, - self.graph.compute_build_order, pool) - - def test_detection_of_cyclic_chunk_dependency_chain(self): - pool = morphlib.sourcepool.SourcePool() - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "first-stratum", - "kind": "stratum", - "sources": [ - { - "name": "first-chunk", - "repo": "repo", - "ref": "original/ref" - }, - { - "name": "second-chunk", - "repo": "repo", - "ref": "original/ref", - "build-depends": [ - "first-chunk" - ] - }, - { - "name": "third-chunk", - "repo": "repo", - "ref": "original/ref", - "build-depends": [ - "second-chunk" - ] - } - ] - } - ''') - first_stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, - 'first-stratum.morph') - pool.add(first_stratum) - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "second-stratum", - "kind": "stratum", - "build-depends": [ - "first-stratum" - ], - "sources": [ - { - "name": "third-chunk", - "repo": "repo", - "ref": "original/ref" - }, - { - "name": "first-chunk", - "repo": "repo", - "ref": "original/ref" - } - ] - } - ''') - second_stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, - 'second-stratum.morph') - pool.add(second_stratum) - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "first-chunk", - "kind": "chunk" - } - ''') - first_chunk = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, - 'first-chunk.morph') - pool.add(first_chunk) - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "second-chunk", - "kind": "chunk" - } - ''') - second_chunk = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, - 'second-chunk.morph') - pool.add(second_chunk) - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "third-chunk", - "kind": "chunk" - } - ''') - second_chunk = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, - 'third-chunk.morph') - pool.add(second_chunk) - - self.assertRaises(morphlib.buildgraph.CyclicDependencyChainError, - self.graph.compute_build_order, pool) - - def test_detection_of_chunk_dependencies_in_invalid_order(self): - pool = morphlib.sourcepool.SourcePool() - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "stratum", - "kind": "stratum", - "sources": [ - { - "name": "first-chunk", - "repo": "repo", - "ref": "original/ref", - "build-depends": [ - "second-chunk" - ] - }, - { - "name": "second-chunk", - "repo": "repo", - "ref": "original/ref" - } - ] - } - ''') - stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'stratum.morph') - pool.add(stratum) - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "first-chunk", - "kind": "chunk" - } - ''') - first_chunk = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'first-chunk.morph') - pool.add(first_chunk) - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "second-chunk", - "kind": "chunk" - } - ''') - second_chunk = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'second-chunk.morph') - pool.add(second_chunk) - - self.assertRaises(morphlib.buildgraph.DependencyOrderError, - self.graph.compute_build_order, pool) - - def test_detection_of_invalid_build_depends_format(self): - pool = morphlib.sourcepool.SourcePool() - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "stratum", - "kind": "stratum", - "sources": [ - { - "name": "chunk", - "repo": "repo", - "ref": "original/ref", - "build-depends": "whatever" - } - ] - } - ''') - stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'stratum.morph') - pool.add(stratum) - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "chunk", - "kind": "chunk" - } - ''') - chunk = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'chunk.morph') - pool.add(chunk) - - self.assertRaises(morphlib.buildgraph.DependencyFormatError, - self.graph.compute_build_order, pool) diff --git a/morphlib/buildorder_tests.py b/morphlib/buildorder_tests.py index 67fbdf4f..32a14120 100644 --- a/morphlib/buildorder_tests.py +++ b/morphlib/buildorder_tests.py @@ -31,7 +31,7 @@ class BuildOrderTests(unittest.TestCase): def test_list_with_one_artifact_results_in_one_group(self): chunk = FakeSource() - artifact = morphlib.artifact.Artifact(chunk, 'chunk', 'key') + artifact = morphlib.artifact.Artifact(chunk, 'chunk') order = morphlib.buildorder.BuildOrder([artifact]) @@ -40,10 +40,10 @@ class BuildOrderTests(unittest.TestCase): def test_list_with_two_unrelated_artifacts_results_in_one_group(self): chunk1 = FakeSource() - artifact1 = morphlib.artifact.Artifact(chunk1, 'chunk1', 'key1') + artifact1 = morphlib.artifact.Artifact(chunk1, 'chunk1') chunk2 = FakeSource() - artifact2 = morphlib.artifact.Artifact(chunk2, 'chunk2', 'key2') + artifact2 = morphlib.artifact.Artifact(chunk2, 'chunk2') order = morphlib.buildorder.BuildOrder([artifact1, artifact2]) @@ -52,10 +52,10 @@ class BuildOrderTests(unittest.TestCase): def test_list_with_two_dependent_artifacts_results_in_two_groups(self): chunk1 = FakeSource() - artifact1 = morphlib.artifact.Artifact(chunk1, 'chunk1', 'key1') + artifact1 = morphlib.artifact.Artifact(chunk1, 'chunk1') chunk2 = FakeSource() - artifact2 = morphlib.artifact.Artifact(chunk2, 'chunk2', 'key2') + artifact2 = morphlib.artifact.Artifact(chunk2, 'chunk2') artifact2.add_dependency(artifact1) order = morphlib.buildorder.BuildOrder([artifact1, artifact2]) @@ -66,14 +66,14 @@ class BuildOrderTests(unittest.TestCase): def test_chain_of_three_dependent_artifacts_results_in_three_groups(self): chunk1 = FakeSource() - artifact1 = morphlib.artifact.Artifact(chunk1, 'chunk1', 'key1') + artifact1 = morphlib.artifact.Artifact(chunk1, 'chunk1') chunk2 = FakeSource() - artifact2 = morphlib.artifact.Artifact(chunk2, 'chunk2', 'key2') + artifact2 = morphlib.artifact.Artifact(chunk2, 'chunk2') artifact2.add_dependency(artifact1) chunk3 = FakeSource() - artifact3 = morphlib.artifact.Artifact(chunk3, 'chunk3', 'key3') + artifact3 = morphlib.artifact.Artifact(chunk3, 'chunk3') artifact3.add_dependency(artifact2) order = morphlib.buildorder.BuildOrder( @@ -86,14 +86,14 @@ class BuildOrderTests(unittest.TestCase): def test_two_artifacts_depending_on_another_results_in_two_groups(self): chunk1 = FakeSource() - artifact1 = morphlib.artifact.Artifact(chunk1, 'chunk1', 'key1') + artifact1 = morphlib.artifact.Artifact(chunk1, 'chunk1') chunk2 = FakeSource() - artifact2 = morphlib.artifact.Artifact(chunk2, 'chunk2', 'key2') + artifact2 = morphlib.artifact.Artifact(chunk2, 'chunk2') artifact2.add_dependency(artifact1) chunk3 = FakeSource() - artifact3 = morphlib.artifact.Artifact(chunk3, 'chunk3', 'key3') + artifact3 = morphlib.artifact.Artifact(chunk3, 'chunk3') artifact3.add_dependency(artifact1) order = morphlib.buildorder.BuildOrder( @@ -105,13 +105,13 @@ class BuildOrderTests(unittest.TestCase): def test_one_artifact_depending_on_two_others_results_in_two_groups(self): chunk1 = FakeSource() - artifact1 = morphlib.artifact.Artifact(chunk1, 'chunk1', 'key1') + artifact1 = morphlib.artifact.Artifact(chunk1, 'chunk1') chunk2 = FakeSource() - artifact2 = morphlib.artifact.Artifact(chunk2, 'chunk2', 'key2') + artifact2 = morphlib.artifact.Artifact(chunk2, 'chunk2') chunk3 = FakeSource() - artifact3 = morphlib.artifact.Artifact(chunk3, 'chunk3', 'key3') + artifact3 = morphlib.artifact.Artifact(chunk3, 'chunk3') artifact3.add_dependency(artifact1) artifact3.add_dependency(artifact2) @@ -124,13 +124,13 @@ class BuildOrderTests(unittest.TestCase): def test_detection_of_cyclic_dependency_chain(self): chunk1 = FakeSource() - artifact1 = morphlib.artifact.Artifact(chunk1, 'chunk1', 'key1') + artifact1 = morphlib.artifact.Artifact(chunk1, 'chunk1') chunk2 = FakeSource() - artifact2 = morphlib.artifact.Artifact(chunk2, 'chunk2', 'key2') + artifact2 = morphlib.artifact.Artifact(chunk2, 'chunk2') chunk3 = FakeSource() - artifact3 = morphlib.artifact.Artifact(chunk3, 'chunk3', 'key3') + artifact3 = morphlib.artifact.Artifact(chunk3, 'chunk3') artifact1.add_dependency(artifact3) artifact2.add_dependency(artifact1) diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index 0a782185..28406b55 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -18,7 +18,8 @@ import hashlib import morphlib -class CacheKeyComputer(): + +class CacheKeyComputer(object): def __init__(self, build_env): self._build_env = build_env @@ -29,8 +30,8 @@ class CacheKeyComputer(): "TOOLCHAIN_TARGET", "PREFIX", "BOOTSTRAP", "CFLAGS")]) - def compute_key(self, source): - return self._hash_id(self.get_cache_id(source)) + def compute_key(self, artifact): + return self._hash_id(self.get_cache_id(artifact)) def _hash_id(self, cache_id): sha = hashlib.sha256() @@ -59,20 +60,19 @@ class CacheKeyComputer(): for item in tup: self._hash_thing(sha, item) - def get_cache_id(self, source): + def get_cache_id(self, artifact): try: - return self._calculated[source] + return self._calculated[artifact] except KeyError: - cacheid = self._calculate(source) - self._calculated[source] = cacheid + cacheid = self._calculate(artifact) + self._calculated[artifact] = cacheid return cacheid - def _calculate(self, source): + def _calculate(self, artifact): return { 'arch': self._build_env.arch, 'env': self._filterenv(self._build_env.env), - 'ref': source.sha1, - 'filename': source.filename, - 'kids': [self.get_cache_id(dependency) - for dependency in source.dependencies], + 'ref': artifact.source.sha1, + 'filename': artifact.source.filename, + 'kids': [self.get_cache_id(x) for x in artifact.dependencies] } diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py index 923cc9f9..51616a12 100644 --- a/morphlib/cachekeycomputer_tests.py +++ b/morphlib/cachekeycomputer_tests.py @@ -18,6 +18,7 @@ import unittest import morphlib + class DummyBuildEnvironment: '''Fake build environment class that doesn't need settings to pick the environment, it just gets passed @@ -27,11 +28,11 @@ class DummyBuildEnvironment: self.arch = morphlib.util.arch() if arch == None else arch self.env = env + class CacheKeyComputerTests(unittest.TestCase): def setUp(self): - pool = morphlib.sourcepool.SourcePool() - self.sources = {} + self.source_pool = morphlib.sourcepool.SourcePool() for name, text in { 'chunk.morph': '''{ "name": "chunk", @@ -83,9 +84,7 @@ class CacheKeyComputerTests(unittest.TestCase): }.iteritems(): source = morphlib.source.Source('repo', 'original/ref', 'sha', morphlib.morph2.Morphology(text), name) - pool.add(source) - self.sources[name] = source - morphlib.buildgraph.BuildGraph().compute_build_order(pool) + self.source_pool.add(source) self.build_env = DummyBuildEnvironment({ "USER": "foouser", "USERNAME": "foouser", @@ -94,8 +93,17 @@ class CacheKeyComputerTests(unittest.TestCase): "PREFIX": "/baserock", "BOOTSTRAP": "false", "CFLAGS": "-O4"}) + self.artifact_resolver = morphlib.artifactresolver.ArtifactResolver() + self.artifacts = self.artifact_resolver.resolve_artifacts( + self.source_pool) self.ckc = morphlib.cachekeycomputer.CacheKeyComputer(self.build_env) + def _find_artifact(self, name): + for artifact in self.artifacts: + if artifact.name == name: + return artifact + raise + def test_compute_key_hashes_all_types(self): runcount = {'thing': 0, 'dict': 0, 'list': 0, 'tuple': 0} def inccount(func, name): @@ -103,11 +111,15 @@ class CacheKeyComputerTests(unittest.TestCase): runcount[name] = runcount[name] + 1 func(sha, item) return f + self.ckc._hash_thing = inccount(self.ckc._hash_thing, 'thing') self.ckc._hash_dict = inccount(self.ckc._hash_dict, 'dict') self.ckc._hash_list = inccount(self.ckc._hash_list, 'list') self.ckc._hash_tuple = inccount(self.ckc._hash_tuple, 'tuple') - self.ckc.compute_key(self.sources['stratum.morph']) + + artifact = self._find_artifact('system') + self.ckc.compute_key(artifact) + self.assertNotEqual(runcount['thing'], 0) self.assertNotEqual(runcount['dict'], 0) self.assertNotEqual(runcount['list'], 0) @@ -115,14 +127,16 @@ class CacheKeyComputerTests(unittest.TestCase): def _valid_sha256(self, s): validchars = '0123456789abcdef' - return len(s) == 64 and all(c in validchars for c in s) + return len(s) == 64 and all([c in validchars for c in s]) def test_compute_key_returns_sha256(self): + artifact = self._find_artifact('system') self.assertTrue(self._valid_sha256( - self.ckc.compute_key(self.sources['system.morph']))) + self.ckc.compute_key(artifact))) def test_different_env_gives_different_key(self): - oldsha = self.ckc.compute_key(self.sources['system.morph']) + artifact = self._find_artifact('system') + oldsha = self.ckc.compute_key(artifact) build_env = DummyBuildEnvironment({ "USER": "foouser", "USERNAME": "foouser", @@ -133,5 +147,4 @@ class CacheKeyComputerTests(unittest.TestCase): "CFLAGS": "-Os"}) ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) - self.assertNotEqual(oldsha, - ckc.compute_key(self.sources['system.morph'])) + self.assertNotEqual(oldsha, ckc.compute_key(artifact)) diff --git a/morphlib/localartifactcache.py b/morphlib/localartifactcache.py index 52b5772c..cf025210 100644 --- a/morphlib/localartifactcache.py +++ b/morphlib/localartifactcache.py @@ -60,17 +60,12 @@ class LocalArtifactCache(object): filename = self._source_metadata_filename(source, cachekey, name) return open(filename) - def _artifact_basename(self, artifact): - return '%s.%s.%s' % (artifact.cache_key, - artifact.source.morphology['kind'], - artifact.name) - def _artifact_filename(self, artifact): - basename = self._artifact_basename(artifact) + basename = artifact.basename() return os.path.join(self.cachedir, basename) def _artifact_metadata_filename(self, artifact, name): - basename = '%s.%s' % (self._artifact_basename(artifact), name) + basename = '%s.%s' % (artifact.basename(), name) return os.path.join(self.cachedir, basename) diff --git a/morphlib/localartifactcache_tests.py b/morphlib/localartifactcache_tests.py index 41eb3d0b..a2fb0810 100644 --- a/morphlib/localartifactcache_tests.py +++ b/morphlib/localartifactcache_tests.py @@ -45,9 +45,9 @@ class LocalArtifactCacheTests(unittest.TestCase): self.source = morphlib.source.Source( 'repo', 'ref', 'sha1', morph, 'chunk.morph') self.runtime_artifact = morphlib.artifact.Artifact( - self.source, 'chunk-runtime', 'cachekey') + self.source, 'chunk-runtime') self.devel_artifact = morphlib.artifact.Artifact( - self.source, 'chunk-devel', 'cachekey') + self.source, 'chunk-devel') def tearDown(self): self.tempdir.remove() |