summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJannis Pohlmann <jannis.pohlmann@codethink.co.uk>2012-04-18 16:49:24 +0100
committerJannis Pohlmann <jannis.pohlmann@codethink.co.uk>2012-04-18 16:49:24 +0100
commit105c0dbbd14626a832ebc036c02bc0c9c60e43bc (patch)
tree9e9398e9e4d4bc8da54a63094142f941b881419e
parent605e21914f8c9536ff42ca31043b26200dab85e2 (diff)
downloadmorph-105c0dbbd14626a832ebc036c02bc0c9c60e43bc.tar.gz
Remove BuildGraph, compute cache keys based on Artifacts.
With this commit, the ArtifactResolver no longer computes the cache keys when creating Artifact objects. This will have to happen as a post-resolving step (e.g. prior to building or checking whether a local or remote artifact cache has any of the resolved artifacts). The CacheKeyComputer now takes an Artifact object and computes the cache keys using its dependencies. BuildGraph is no longer needed for the CacheKeyComputer unit tests.
-rwxr-xr-xmorph4
-rw-r--r--morphlib/__init__.py1
-rw-r--r--morphlib/artifact.py4
-rw-r--r--morphlib/artifact_tests.py11
-rw-r--r--morphlib/artifactresolver.py29
-rw-r--r--morphlib/artifactresolver_tests.py42
-rw-r--r--morphlib/buildgraph.py208
-rw-r--r--morphlib/buildgraph_tests.py851
-rw-r--r--morphlib/buildorder_tests.py34
-rw-r--r--morphlib/cachekeycomputer.py24
-rw-r--r--morphlib/cachekeycomputer_tests.py35
-rw-r--r--morphlib/localartifactcache.py9
-rw-r--r--morphlib/localartifactcache_tests.py4
13 files changed, 76 insertions, 1180 deletions
diff --git a/morph b/morph
index 1d6dc4cb..0b1e57eb 100755
--- a/morph
+++ b/morph
@@ -259,9 +259,7 @@ class Morph(cliapp.Application):
local_repo_cache, remote_repo_cache,
repo, ref, filename)
- env = morphlib.buildenvironment.BuildEnvironment(self.settings)
- computer = morphlib.cachekeycomputer.CacheKeyComputer(env)
- resolver = morphlib.artifactresolver.ArtifactResolver(computer)
+ resolver = morphlib.artifactresolver.ArtifactResolver()
artifacts = resolver.resolve_artifacts(pool)
self.output.write('dependency graph for %s|%s|%s:\n' %
diff --git a/morphlib/__init__.py b/morphlib/__init__.py
index be3d7c5c..44b6ce53 100644
--- a/morphlib/__init__.py
+++ b/morphlib/__init__.py
@@ -23,7 +23,6 @@ import bins
import blobs
import buildcontroller
import builddependencygraph
-import buildgraph
import buildenvironment
import buildorder
import buildsystem
diff --git a/morphlib/artifact.py b/morphlib/artifact.py
index f5c2e8a5..3a438747 100644
--- a/morphlib/artifact.py
+++ b/morphlib/artifact.py
@@ -31,10 +31,10 @@ class Artifact(object):
'''
- def __init__(self, source, name, cache_key):
+ def __init__(self, source, name):
self.source = source
self.name = name
- self.cache_key = cache_key
+ self.cache_key = None
self.dependencies = []
self.dependents = []
diff --git a/morphlib/artifact_tests.py b/morphlib/artifact_tests.py
index faeef879..5c912cd7 100644
--- a/morphlib/artifact_tests.py
+++ b/morphlib/artifact_tests.py
@@ -27,7 +27,7 @@ class ArtifactTests(unittest.TestCase):
{
"chunk": "chunk",
"kind": "chunk",
- "artifacts": {
+ "chunks": {
"chunk-runtime": [
"usr/bin",
"usr/sbin",
@@ -42,12 +42,11 @@ class ArtifactTests(unittest.TestCase):
''')
self.source = morphlib.source.Source(
'repo', 'ref', 'sha1', morph, 'chunk.morph')
- self.cache_key = 'cachekey'
self.artifact_name = 'chunk-runtime'
self.artifact = morphlib.artifact.Artifact(
- self.source, self.artifact_name, self.cache_key)
+ self.source, self.artifact_name)
self.other = morphlib.artifact.Artifact(
- self.source, self.artifact_name, self.cache_key)
+ self.source, self.artifact_name)
def test_constructor_sets_source(self):
self.assertEqual(self.artifact.source, self.source)
@@ -55,8 +54,8 @@ class ArtifactTests(unittest.TestCase):
def test_constructor_sets_name(self):
self.assertEqual(self.artifact.name, self.artifact_name)
- def test_constructor_sets_cache_key(self):
- self.assertEqual(self.artifact.cache_key, self.cache_key)
+ def test_constructor_initializes_cache_key_as_none(self):
+ self.assertEqual(self.artifact.cache_key, None)
def test_sets_dependencies_to_empty(self):
self.assertEqual(self.artifact.dependencies, [])
diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py
index df3fdf87..25836b41 100644
--- a/morphlib/artifactresolver.py
+++ b/morphlib/artifactresolver.py
@@ -70,8 +70,7 @@ class ArtifactResolver(object):
'''
- def __init__(self, cache_key_computer):
- self.cache_key_computer = cache_key_computer
+ def __init__(self):
self._cached_artifacts = None
self._added_artifacts = None
self._source_pool = None
@@ -93,11 +92,9 @@ class ArtifactResolver(object):
while queue:
source = queue.popleft()
- cache_key = self.cache_key_computer.compute_key(source)
-
if source.morphology['kind'] == 'system':
artifact = self._get_artifact(
- source, source.morphology['name'], cache_key)
+ source, source.morphology['name'])
if not artifact in self._added_artifacts:
artifacts.append(artifact)
@@ -112,7 +109,7 @@ class ArtifactResolver(object):
self._added_artifacts.add(artifact)
elif source.morphology['kind'] == 'stratum':
artifact = self._get_artifact(
- source, source.morphology['name'], cache_key)
+ source, source.morphology['name'])
if not artifact in self._added_artifacts:
artifacts.append(artifact)
@@ -128,7 +125,7 @@ class ArtifactResolver(object):
elif source.morphology['kind'] == 'chunk':
names = self._chunk_artifact_names(source)
for name in names:
- artifact = self._get_artifact(source, name, cache_key)
+ artifact = self._get_artifact(source, name)
if not artifact in self._added_artifacts:
artifacts.append(artifact)
self._added_artifacts.add(artifact)
@@ -143,12 +140,12 @@ class ArtifactResolver(object):
if x.morphology['kind'] != 'chunk']
return collections.deque(sources)
- def _get_artifact(self, source, name, cache_key):
- info = (source, name, cache_key)
+ def _get_artifact(self, source, name):
+ info = (source, name)
if info in self._cached_artifacts:
return self._cached_artifacts[info]
else:
- artifact = morphlib.artifact.Artifact(info[0], info[1], info[2])
+ artifact = morphlib.artifact.Artifact(info[0], info[1])
self._cached_artifacts[info] = artifact
return artifact
@@ -161,8 +158,7 @@ class ArtifactResolver(object):
system.source.original_ref,
'%s.morph' % stratum_name)
- cache_key = self.cache_key_computer.compute_key(source)
- stratum = self._get_artifact(source, stratum_name, cache_key)
+ stratum = self._get_artifact(source, stratum_name)
system.add_dependency(stratum)
queue.append(source)
@@ -183,9 +179,7 @@ class ArtifactResolver(object):
stratum.source.original_ref,
'%s.morph' % stratum_name)
- cache_key = self.cache_key_computer.compute_key(other_source)
- other_stratum = self._get_artifact(
- other_source, stratum_name, cache_key)
+ other_stratum = self._get_artifact(other_source, stratum_name)
strata.append(other_stratum)
@@ -211,10 +205,7 @@ class ArtifactResolver(object):
if not info['name'] in possible_names:
raise UndefinedChunkArtifactError(stratum.source, info['name'])
- cache_key = self.cache_key_computer.compute_key(chunk_source)
-
- chunk_artifact = self._get_artifact(
- chunk_source, info['name'], cache_key)
+ chunk_artifact = self._get_artifact(chunk_source, info['name'])
chunk_artifacts.append(chunk_artifact)
artifacts.append(chunk_artifact)
diff --git a/morphlib/artifactresolver_tests.py b/morphlib/artifactresolver_tests.py
index d77ff5bb..467c882d 100644
--- a/morphlib/artifactresolver_tests.py
+++ b/morphlib/artifactresolver_tests.py
@@ -20,13 +20,6 @@ import unittest
import morphlib
-class FakeCacheKeyComputer(object):
- '''Fake computer that uses the uppercase source name as the cache key.'''
-
- def compute_key(self, source):
- return source.morphology['name'].upper()
-
-
class FakeChunkMorphology(morphlib.morph2.Morphology):
def __init__(self, name, artifact_names=[]):
@@ -94,9 +87,7 @@ class FakeStratumMorphology(morphlib.morph2.Morphology):
class ArtifactResolverTests(unittest.TestCase):
def setUp(self):
- self.cache_key_computer = FakeCacheKeyComputer()
- self.resolver = morphlib.artifactresolver.ArtifactResolver(
- self.cache_key_computer)
+ self.resolver = morphlib.artifactresolver.ArtifactResolver()
def test_resolve_artifacts_using_an_empty_pool(self):
pool = morphlib.sourcepool.SourcePool()
@@ -117,7 +108,6 @@ class ArtifactResolverTests(unittest.TestCase):
self.assertEqual(artifacts[0].source, source)
self.assertEqual(artifacts[0].name, 'chunk')
- self.assertEqual(artifacts[0].cache_key, 'CHUNK')
self.assertEqual(artifacts[0].dependencies, [])
self.assertEqual(artifacts[0].dependents, [])
@@ -134,7 +124,6 @@ class ArtifactResolverTests(unittest.TestCase):
self.assertEqual(len(artifacts), 1)
self.assertEqual(artifacts[0].source, source)
self.assertEqual(artifacts[0].name, 'chunk-runtime')
- self.assertEqual(artifacts[0].cache_key, 'CHUNK')
self.assertEqual(artifacts[0].dependencies, [])
self.assertEqual(artifacts[0].dependents, [])
@@ -152,13 +141,11 @@ class ArtifactResolverTests(unittest.TestCase):
self.assertEqual(artifacts[0].source, source)
self.assertEqual(artifacts[0].name, 'chunk-devel')
- self.assertEqual(artifacts[0].cache_key, 'CHUNK')
self.assertEqual(artifacts[0].dependencies, [])
self.assertEqual(artifacts[0].dependents, [])
self.assertEqual(artifacts[1].source, source)
self.assertEqual(artifacts[1].name, 'chunk-runtime')
- self.assertEqual(artifacts[1].cache_key, 'CHUNK')
self.assertEqual(artifacts[1].dependencies, [])
self.assertEqual(artifacts[1].dependents, [])
@@ -180,7 +167,6 @@ class ArtifactResolverTests(unittest.TestCase):
self.assertEqual(artifacts[0].source, stratum)
self.assertEqual(artifacts[0].name, 'foo')
- self.assertEqual(artifacts[0].cache_key, 'FOO')
self.assertEqual(artifacts[0].dependencies, [])
self.assertEqual(artifacts[0].dependents, [])
@@ -202,7 +188,6 @@ class ArtifactResolverTests(unittest.TestCase):
self.assertEqual(artifacts[0].source, system)
self.assertEqual(artifacts[0].name, 'foo')
- self.assertEqual(artifacts[0].cache_key, 'FOO')
self.assertEqual(artifacts[0].dependencies, [])
self.assertEqual(artifacts[0].dependents, [])
@@ -226,13 +211,11 @@ class ArtifactResolverTests(unittest.TestCase):
self.assertEqual(artifacts[0].source, stratum)
self.assertEqual(artifacts[0].name, 'stratum')
- self.assertEqual(artifacts[0].cache_key, 'STRATUM')
self.assertEqual(artifacts[0].dependencies, [artifacts[1]])
self.assertEqual(artifacts[0].dependents, [])
self.assertEqual(artifacts[1].source, chunk)
self.assertEqual(artifacts[1].name, 'chunk')
- self.assertEqual(artifacts[1].cache_key, 'CHUNK')
self.assertEqual(artifacts[1].dependencies, [])
self.assertEqual(artifacts[1].dependents, [artifacts[0]])
@@ -259,20 +242,17 @@ class ArtifactResolverTests(unittest.TestCase):
self.assertEqual(artifacts[0].source, stratum)
self.assertEqual(artifacts[0].name, 'stratum')
- self.assertEqual(artifacts[0].cache_key, 'STRATUM')
self.assertEqual(artifacts[0].dependencies,
[artifacts[1], artifacts[2]])
self.assertEqual(artifacts[0].dependents, [])
self.assertEqual(artifacts[1].source, chunk)
self.assertEqual(artifacts[1].name, 'chunk-devel')
- self.assertEqual(artifacts[1].cache_key, 'CHUNK')
self.assertEqual(artifacts[1].dependencies, [])
self.assertEqual(artifacts[1].dependents, [artifacts[0], artifacts[2]])
self.assertEqual(artifacts[2].source, chunk)
self.assertEqual(artifacts[2].name, 'chunk-runtime')
- self.assertEqual(artifacts[2].cache_key, 'CHUNK')
self.assertEqual(artifacts[2].dependencies, [artifacts[1]])
self.assertEqual(artifacts[2].dependents, [artifacts[0]])
@@ -298,13 +278,11 @@ class ArtifactResolverTests(unittest.TestCase):
self.assertEqual(artifacts[0].source, stratum)
self.assertEqual(artifacts[0].name, 'stratum')
- self.assertEqual(artifacts[0].cache_key, 'STRATUM')
self.assertEqual(artifacts[0].dependencies, [artifacts[1]])
self.assertEqual(artifacts[0].dependents, [])
self.assertEqual(artifacts[1].source, chunk)
self.assertEqual(artifacts[1].name, 'chunk-runtime')
- self.assertEqual(artifacts[1].cache_key, 'CHUNK')
self.assertEqual(artifacts[1].dependencies, [])
self.assertEqual(artifacts[1].dependents, [artifacts[0]])
@@ -336,20 +314,17 @@ class ArtifactResolverTests(unittest.TestCase):
self.assertEqual(artifacts[0].source, stratum)
self.assertEqual(artifacts[0].name, 'stratum')
- self.assertEqual(artifacts[0].cache_key, 'STRATUM')
self.assertEqual(artifacts[0].dependencies,
[artifacts[1], artifacts[2]])
self.assertEqual(artifacts[0].dependents, [])
self.assertEqual(artifacts[1].source, foo_chunk)
self.assertEqual(artifacts[1].name, 'foo')
- self.assertEqual(artifacts[1].cache_key, 'FOO')
self.assertEqual(artifacts[1].dependencies, [])
self.assertEqual(artifacts[1].dependents, [artifacts[0], artifacts[2]])
self.assertEqual(artifacts[2].source, bar_chunk)
self.assertEqual(artifacts[2].name, 'bar')
- self.assertEqual(artifacts[2].cache_key, 'BAR')
self.assertEqual(artifacts[2].dependencies, [artifacts[1]])
self.assertEqual(artifacts[2].dependents, [artifacts[0]])
@@ -372,13 +347,11 @@ class ArtifactResolverTests(unittest.TestCase):
self.assertEqual(artifacts[0].source, stratum1)
self.assertEqual(artifacts[0].name, 'stratum1')
- self.assertEqual(artifacts[0].cache_key, 'STRATUM1')
self.assertEqual(artifacts[0].dependencies, [])
self.assertEqual(artifacts[0].dependents, [artifacts[1]])
self.assertEqual(artifacts[1].source, stratum2)
self.assertEqual(artifacts[1].name, 'stratum2')
- self.assertEqual(artifacts[1].cache_key, 'STRATUM2')
self.assertEqual(artifacts[1].dependencies, [artifacts[0]])
self.assertEqual(artifacts[1].dependents, [])
@@ -415,27 +388,23 @@ class ArtifactResolverTests(unittest.TestCase):
self.assertEqual(artifacts[0].source, stratum1)
self.assertEqual(artifacts[0].name, 'stratum1')
- self.assertEqual(artifacts[0].cache_key, 'STRATUM1')
self.assertEqual(artifacts[0].dependencies, [])
self.assertEqual(artifacts[0].dependents,
[artifacts[1], artifacts[2], artifacts[3]])
self.assertEqual(artifacts[1].source, stratum2)
self.assertEqual(artifacts[1].name, 'stratum2')
- self.assertEqual(artifacts[1].cache_key, 'STRATUM2')
self.assertEqual(artifacts[1].dependencies,
[artifacts[0], artifacts[2], artifacts[3]])
self.assertEqual(artifacts[1].dependents, [])
self.assertEqual(artifacts[2].source, chunk1)
self.assertEqual(artifacts[2].name, 'chunk1')
- self.assertEqual(artifacts[2].cache_key, 'CHUNK1')
self.assertEqual(artifacts[2].dependencies, [artifacts[0]])
self.assertEqual(artifacts[2].dependents, [artifacts[1], artifacts[3]])
self.assertEqual(artifacts[3].source, chunk2)
self.assertEqual(artifacts[3].name, 'chunk2')
- self.assertEqual(artifacts[3].cache_key, 'CHUNK2')
self.assertEqual(artifacts[3].dependencies,
[artifacts[0], artifacts[2]])
self.assertEqual(artifacts[3].dependents, [artifacts[1]])
@@ -474,20 +443,17 @@ class ArtifactResolverTests(unittest.TestCase):
self.assertEqual(artifacts[0].source, stratum1)
self.assertEqual(artifacts[0].name, 'stratum1')
- self.assertEqual(artifacts[0].cache_key, 'STRATUM1')
self.assertEqual(artifacts[0].dependencies, [])
self.assertEqual(artifacts[0].dependents, [artifacts[1], artifacts[2]])
self.assertEqual(artifacts[1].source, system)
self.assertEqual(artifacts[1].name, 'system')
- self.assertEqual(artifacts[1].cache_key, 'SYSTEM')
self.assertEqual(artifacts[1].dependencies,
[artifacts[0], artifacts[2]])
self.assertEqual(artifacts[1].dependents, [])
self.assertEqual(artifacts[2].source, stratum2)
self.assertEqual(artifacts[2].name, 'stratum2')
- self.assertEqual(artifacts[2].cache_key, 'STRATUM2')
self.assertEqual(artifacts[2].dependencies, [artifacts[0]])
self.assertEqual(artifacts[2].dependents, [artifacts[1]])
@@ -549,27 +515,23 @@ class ArtifactResolverTests(unittest.TestCase):
self.assertEqual(artifacts[0].source, stratum)
self.assertEqual(artifacts[0].name, 'stratum')
- self.assertEqual(artifacts[0].cache_key, 'STRATUM')
self.assertEqual(artifacts[0].dependencies,
[artifacts[1], artifacts[2], artifacts[3]])
self.assertEqual(artifacts[0].dependents, [])
self.assertEqual(artifacts[1].source, chunk1)
self.assertEqual(artifacts[1].name, 'chunk1')
- self.assertEqual(artifacts[1].cache_key, 'CHUNK1')
self.assertEqual(artifacts[1].dependencies, [])
self.assertEqual(artifacts[1].dependents,
[artifacts[0], artifacts[3]])
self.assertEqual(artifacts[2].source, chunk2)
self.assertEqual(artifacts[2].name, 'chunk2')
- self.assertEqual(artifacts[2].cache_key, 'CHUNK2')
self.assertEqual(artifacts[2].dependencies, [])
self.assertEqual(artifacts[2].dependents, [artifacts[0], artifacts[3]])
self.assertEqual(artifacts[3].source, chunk3)
self.assertEqual(artifacts[3].name, 'chunk3')
- self.assertEqual(artifacts[3].cache_key, 'CHUNK3')
self.assertEqual(artifacts[3].dependencies,
[artifacts[1], artifacts[2]])
self.assertEqual(artifacts[3].dependents, [artifacts[0]])
@@ -689,13 +651,11 @@ class ArtifactResolverTests(unittest.TestCase):
self.assertEqual(artifacts[0].source, stratum)
self.assertEqual(artifacts[0].name, 'stratum')
- self.assertEqual(artifacts[0].cache_key, 'STRATUM')
self.assertEqual(artifacts[0].dependencies, [artifacts[1]])
self.assertEqual(artifacts[0].dependents, [])
self.assertEqual(artifacts[1].source, chunk)
self.assertEqual(artifacts[1].name, 'chunk')
- self.assertEqual(artifacts[1].cache_key, 'CHUNK')
self.assertEqual(artifacts[1].dependencies, [])
self.assertEqual(artifacts[1].dependents, [artifacts[0]])
diff --git a/morphlib/buildgraph.py b/morphlib/buildgraph.py
deleted file mode 100644
index 687dbb13..00000000
--- a/morphlib/buildgraph.py
+++ /dev/null
@@ -1,208 +0,0 @@
-# Copyright (C) 2012 Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import cliapp
-import collections
-
-
-class MutualDependencyError(cliapp.AppException):
-
- def __init__(self, a, b):
- cliapp.AppException.__init__(
- self, 'Cyclic dependency between %s and %s detected' % (a, b))
-
-
-class CyclicDependencyChainError(cliapp.AppException):
-
- def __init__(self):
- cliapp.AppException.__init__(
- self, 'Cyclic dependency detected somewhere')
-
-
-class DependencyOrderError(cliapp.AppException):
-
- def __init__(self, stratum, chunk, dependency_name):
- cliapp.AppException.__init__(
- self, 'In stratum %s, chunk %s references its dependency %s '
- 'before it is defined' % (stratum, chunk, dependency_name))
-
-
-class DependencyFormatError(cliapp.AppException):
-
- def __init__(self, stratum, chunk):
- cliapp.AppException.__init__(
- self, 'In stratum %s, chunk %s uses an invalid '
- 'build-depends format' % (stratum, chunk))
-
-
-class BuildGraph(object):
-
- def compute_build_order(self, source_pool):
- self._realise_dependencies(source_pool)
- sorting = self._compute_topological_sorting(source_pool)
- groups = self._create_build_groups(sorting)
- return groups
-
- def _realise_dependencies(self, source_pool):
- queue = collections.deque(source_pool)
- while queue:
- source = queue.popleft()
-
- if source.morphology['kind'] == 'system':
- self._realise_system_dependencies(source, queue, source_pool)
- elif source.morphology['kind'] == 'stratum':
- self._realise_stratum_dependencies(source, queue, source_pool)
-
- def _realise_system_dependencies(self, system, queue, source_pool):
- for stratum_name in system.morphology['strata']:
- stratum = source_pool.lookup(
- system.repo_name,
- system.original_ref,
- '%s.morph' % stratum_name)
-
- system.add_dependency(stratum)
- queue.append(stratum)
-
- def _realise_stratum_dependencies(self, stratum, queue, source_pool):
- strata = []
-
- if stratum.morphology['build-depends']:
- for stratum_name in stratum.morphology['build-depends']:
- other_stratum = source_pool.lookup(
- stratum.repo_name,
- stratum.original_ref,
- '%s.morph' % stratum_name)
- strata.append(other_stratum)
-
- if other_stratum.depends_on(stratum):
- raise MutualDependencyError(stratum, other_stratum)
-
- stratum.add_dependency(other_stratum)
- queue.append(other_stratum)
-
- chunks = []
- processed_chunks = []
- name_to_processed_chunk = {}
-
- for info in stratum.morphology['sources']:
- chunk = source_pool.lookup(
- info['repo'],
- info['ref'],
- '%s.morph' % info['morph'])
- chunks.append(chunk)
-
- stratum.add_dependency(chunk)
-
- for other_stratum in strata:
- chunk.add_dependency(other_stratum)
-
- build_depends = info.get('build-depends', None)
-
- if build_depends is None:
- for earlier_chunk in processed_chunks:
- if earlier_chunk.depends_on(chunk):
- raise MutualDependencyError(chunk, earlier_chunk)
- chunk.add_dependency(earlier_chunk)
- elif isinstance(build_depends, list):
- for name in build_depends:
- other_chunk = name_to_processed_chunk.get(name, None)
- if other_chunk:
- chunk.add_dependency(other_chunk)
- else:
- raise DependencyOrderError(stratum, info['name'], name)
- else:
- raise DependencyFormatError(stratum, info['name'])
- processed_chunks.append(chunk)
- name_to_processed_chunk[info['name']] = chunk
-
- def _compute_topological_sorting(self, source_pool):
- '''Computes a topological sorting of the build graph.
-
- A topological sorting basically is the result of a series of
- breadth-first searches starting at each leaf node (sources with no
- dependencies). Sources are added to the sorting as soon as all their
- dependencies have been added (which means that by then, all
- dependencies are satisfied).
-
- For more information, see
- http://en.wikipedia.org/wiki/Topological_sorting.
-
- '''
-
- # map sources to sets of satisfied dependencies. this is to detect when
- # we can actually add sources to the BFS queue. rather than dropping
- # links between nodes, like most topological sorting algorithms do,
- # we simply remember all satisfied dependencies and check if all
- # of them are met repeatedly
- satisfied_dependencies = {}
-
- # create an empty sorting
- sorting = collections.deque()
-
- # create a set of leafs to start the DFS from
- leafs = collections.deque()
- for source in source_pool:
- satisfied_dependencies[source] = set()
- if len(source.dependencies) == 0:
- leafs.append(source)
-
- while len(leafs) > 0:
- # fetch a leaf source from the DFS queue
- source = leafs.popleft()
-
- # add it to the sorting
- sorting.append(source)
-
- # mark this dependency as resolved
- for dependent in source.dependents:
- satisfied_dependencies[dependent].add(source)
-
- # add the dependent blob as a leaf if all
- # its dependencies have been resolved
- has = len(satisfied_dependencies[dependent])
- needs = len(dependent.dependencies)
- if has == needs:
- leafs.append(dependent)
-
- # if not all dependencies were resolved on the way, we
- # have found at least one cyclic dependency
- if len(sorting) < len(source_pool):
- raise CyclicDependencyChainError()
-
- return sorting
-
- def _create_build_groups(self, sorting):
- groups = collections.deque()
-
- if sorting:
- # create the first group
- group = []
- groups.append(group)
-
- # traverse the build graph in topological order
- for source in sorting:
- # add the current item to the current group, or a new group
- # if one of its dependencies is in the current one
- create_group = False
- for dependency in source.dependencies:
- if dependency in group:
- create_group = True
- if create_group:
- group = []
- groups.append(group)
- group.append(source)
-
- return groups
diff --git a/morphlib/buildgraph_tests.py b/morphlib/buildgraph_tests.py
deleted file mode 100644
index 18abcb6c..00000000
--- a/morphlib/buildgraph_tests.py
+++ /dev/null
@@ -1,851 +0,0 @@
-# Copyright (C) 2012 Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import collections
-import unittest
-
-import morphlib.buildgraph
-import morphlib.source
-
-
-class BuildGraphTests(unittest.TestCase):
-
- def setUp(self):
- self.graph = morphlib.buildgraph.BuildGraph()
-
- def test_create_empty_build_order_for_empty_pool(self):
- pool = morphlib.sourcepool.SourcePool()
- order = self.graph.compute_build_order(pool)
- self.assertEqual(order, collections.deque())
-
- def test_build_order_with_a_single_chunk(self):
- pool = morphlib.sourcepool.SourcePool()
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "foo",
- "kind": "chunk",
- "artifacts": {
- "foo-runtime": [ "usr/bin" ],
- "foo-devel": [ "usr/lib" ]
- }
- }
- ''')
- source = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph, 'foo.morph')
- pool.add(source)
-
- order = self.graph.compute_build_order(pool)
- desired_order = collections.deque([
- [source]
- ])
- self.assertEqual(order, desired_order)
-
- def test_build_order_with_a_single_empty_stratum(self):
- pool = morphlib.sourcepool.SourcePool()
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "foo",
- "kind": "stratum"
- }
- ''')
- source = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph, 'foo.morph')
- pool.add(source)
-
- order = self.graph.compute_build_order(pool)
- desired_order = collections.deque([
- [source]
- ])
- self.assertEqual(order, desired_order)
-
- def test_build_order_with_a_single_empty_system(self):
- pool = morphlib.sourcepool.SourcePool()
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "foo",
- "kind": "system"
- }
- ''')
- source = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph, 'foo.morph')
- pool.add(source)
-
- order = self.graph.compute_build_order(pool)
- desired_order = collections.deque([
- [source]
- ])
- self.assertEqual(order, desired_order)
-
- def test_build_order_with_a_one_chunk_stratum(self):
- pool = morphlib.sourcepool.SourcePool()
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "stratum",
- "kind": "stratum",
- "sources": [
- {
- "name": "chunk",
- "repo": "repo",
- "ref": "original/ref"
- }
- ]
- }
- ''')
- stratum = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph, 'stratum.morph')
- pool.add(stratum)
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "chunk",
- "kind": "chunk",
- "artifacts": {
- "foo-runtime": [ "usr/bin" ],
- "foo-devel": [ "usr/lib" ]
- }
- }
- ''')
- chunk = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph, 'chunk.morph')
- pool.add(chunk)
-
- order = self.graph.compute_build_order(pool)
- desired_order = collections.deque([
- [chunk],
- [stratum]
- ])
- self.assertEqual(order, desired_order)
-
- self.assertEqual(stratum.dependencies, [chunk])
-
- def test_build_order_with_a_one_chunk_artifact_stratum(self):
- pool = morphlib.sourcepool.SourcePool()
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "stratum",
- "kind": "stratum",
- "sources": [
- {
- "name": "chunk-runtime",
- "morph": "chunk",
- "repo": "repo",
- "ref": "original/ref"
- }
- ]
- }
- ''')
- stratum = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph, 'stratum.morph')
- pool.add(stratum)
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "chunk",
- "kind": "chunk",
- "artifacts": {
- "foo-runtime": [ "usr/bin" ],
- "foo-devel": [ "usr/lib" ]
- }
- }
- ''')
- chunk = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph, 'chunk.morph')
- pool.add(chunk)
-
- order = self.graph.compute_build_order(pool)
- desired_order = collections.deque([
- [chunk],
- [stratum]
- ])
- self.assertEqual(order, desired_order)
-
- self.assertEqual(stratum.dependencies, [chunk])
-
- def test_build_order_with_stratum_and_implicit_dependencies(self):
- pool = morphlib.sourcepool.SourcePool()
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "stratum",
- "kind": "stratum",
- "sources": [
- {
- "name": "first-chunk",
- "repo": "repo",
- "ref": "original/ref"
- },
- {
- "name": "second-chunk",
- "repo": "repo",
- "ref": "original/ref"
- },
- {
- "name": "third-chunk",
- "repo": "repo",
- "ref": "original/ref"
- }
- ]
- }
- ''')
- stratum = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph, 'stratum.morph')
- pool.add(stratum)
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "first-chunk",
- "kind": "chunk"
- }
- ''')
- first_chunk = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph, 'first-chunk.morph')
- pool.add(first_chunk)
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "second-chunk",
- "kind": "chunk"
- }
- ''')
- second_chunk = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph, 'second-chunk.morph')
- pool.add(second_chunk)
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "third-chunk",
- "kind": "chunk"
- }
- ''')
- third_chunk = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph, 'third-chunk.morph')
- pool.add(third_chunk)
-
- order = self.graph.compute_build_order(pool)
- desired_order = collections.deque([
- [first_chunk],
- [second_chunk],
- [third_chunk],
- [stratum]
- ])
- self.assertEqual(order, desired_order)
-
- self.assertEqual(first_chunk.dependencies, [])
- self.assertEqual(second_chunk.dependencies, [first_chunk])
- self.assertEqual(third_chunk.dependencies, [first_chunk, second_chunk])
- self.assertEqual(stratum.dependencies,
- [first_chunk, second_chunk, third_chunk])
-
- def test_build_order_with_explicit_dependencies(self):
- pool = morphlib.sourcepool.SourcePool()
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "stratum",
- "kind": "stratum",
- "sources": [
- {
- "name": "first-chunk",
- "repo": "repo",
- "ref": "original/ref",
- "build-depends": []
- },
- {
- "name": "second-chunk",
- "repo": "repo",
- "ref": "original/ref",
- "build-depends": []
- },
- {
- "name": "third-chunk",
- "repo": "repo",
- "ref": "original/ref",
- "build-depends": [
- "first-chunk",
- "second-chunk"
- ]
- }
- ]
- }
- ''')
- stratum = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph, 'stratum.morph')
- pool.add(stratum)
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "first-chunk",
- "kind": "chunk"
- }
- ''')
- first_chunk = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph, 'first-chunk.morph')
- pool.add(first_chunk)
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "second-chunk",
- "kind": "chunk"
- }
- ''')
- second_chunk = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph, 'second-chunk.morph')
- pool.add(second_chunk)
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "third-chunk",
- "kind": "chunk"
- }
- ''')
- third_chunk = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph, 'third-chunk.morph')
- pool.add(third_chunk)
-
- order = self.graph.compute_build_order(pool)
- desired_order = collections.deque([
- [first_chunk, second_chunk],
- [third_chunk],
- [stratum]
- ])
- self.assertEqual(order, desired_order)
-
- self.assertEqual(first_chunk.dependencies, [])
- self.assertEqual(second_chunk.dependencies, [])
- self.assertEqual(third_chunk.dependencies, [first_chunk, second_chunk])
- self.assertEqual(stratum.dependencies,
- [first_chunk, second_chunk, third_chunk])
-
- def test_build_order_with_stratum_dependencies(self):
- pool = morphlib.sourcepool.SourcePool()
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "first-stratum",
- "kind": "stratum"
- }
- ''')
- first_stratum = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph,
- 'first-stratum.morph')
- pool.add(first_stratum)
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "second-stratum",
- "kind": "stratum",
- "build-depends": [
- "first-stratum"
- ]
- }
- ''')
- second_stratum = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph,
- 'second-stratum.morph')
- pool.add(second_stratum)
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "third-stratum",
- "kind": "stratum",
- "build-depends": [
- "second-stratum"
- ]
- }
- ''')
- third_stratum = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph,
- 'third-stratum.morph')
- pool.add(third_stratum)
-
- order = self.graph.compute_build_order(pool)
- desired_order = collections.deque([
- [first_stratum],
- [second_stratum],
- [third_stratum]
- ])
- self.assertEqual(order, desired_order)
-
- self.assertEqual(first_stratum.dependencies, [])
- self.assertEqual(second_stratum.dependencies, [first_stratum])
- self.assertEqual(third_stratum.dependencies, [second_stratum])
-
- def test_build_order_with_stratum_and_chunk_dependencies(self):
- pool = morphlib.sourcepool.SourcePool()
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "first-stratum",
- "kind": "stratum"
- }
- ''')
- first_stratum = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph,
- 'first-stratum.morph')
- pool.add(first_stratum)
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "second-stratum",
- "kind": "stratum",
- "build-depends": [
- "first-stratum"
- ],
- "sources": [
- {
- "name": "first-chunk",
- "repo": "repo",
- "ref": "original/ref"
- },
- {
- "name": "second-chunk",
- "repo": "repo",
- "ref": "original/ref"
- }
- ]
- }
- ''')
- second_stratum = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph,
- 'second-stratum.morph')
- pool.add(second_stratum)
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "first-chunk",
- "kind": "chunk"
- }
- ''')
- first_chunk = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph, 'first-chunk.morph')
- pool.add(first_chunk)
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "second-chunk",
- "kind": "chunk"
- }
- ''')
- second_chunk = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph, 'second-chunk.morph')
- pool.add(second_chunk)
-
- order = self.graph.compute_build_order(pool)
- desired_order = collections.deque([
- [first_stratum],
- [first_chunk],
- [second_chunk],
- [second_stratum]
- ])
- self.assertEqual(order, desired_order)
-
- self.assertEqual(first_stratum.dependencies, [])
- self.assertEqual(first_chunk.dependencies, [first_stratum])
- self.assertEqual(second_chunk.dependencies,
- [first_stratum, first_chunk])
- self.assertEqual(second_stratum.dependencies,
- [first_stratum, first_chunk, second_chunk])
-
- def test_build_order_with_a_system_and_two_strata(self):
- pool = morphlib.sourcepool.SourcePool()
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "first-stratum",
- "kind": "stratum"
- }
- ''')
- first_stratum = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph,
- 'first-stratum.morph')
- pool.add(first_stratum)
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "second-stratum",
- "kind": "stratum"
- }
- ''')
- second_stratum = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph,
- 'second-stratum.morph')
- pool.add(second_stratum)
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "system",
- "kind": "system",
- "strata": [
- "first-stratum",
- "second-stratum"
- ]
- }
- ''')
- system = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph, 'system.morph')
- pool.add(system)
-
- order = self.graph.compute_build_order(pool)
- desired_order = collections.deque([
- [first_stratum, second_stratum],
- [system]
- ])
- self.assertEqual(order, desired_order)
-
- self.assertEqual(first_stratum.dependencies, [])
- self.assertEqual(second_stratum.dependencies, [])
- self.assertEqual(system.dependencies, [first_stratum, second_stratum])
-
- def test_detection_of_mutual_dependency_between_two_strata(self):
- pool = morphlib.sourcepool.SourcePool()
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "first-stratum",
- "kind": "stratum",
- "build-depends": [
- "second-stratum"
- ]
- }
- ''')
- first_stratum = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph,
- 'first-stratum.morph')
- pool.add(first_stratum)
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "second-stratum",
- "kind": "stratum",
- "build-depends": [
- "first-stratum"
- ]
- }
- ''')
- second_stratum = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph,
- 'second-stratum.morph')
- pool.add(second_stratum)
-
- self.assertRaises(morphlib.buildgraph.MutualDependencyError,
- self.graph.compute_build_order, pool)
-
- def test_detection_of_mutual_dependency_between_consecutive_chunks(self):
- pool = morphlib.sourcepool.SourcePool()
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "first-stratum",
- "kind": "stratum",
- "sources": [
- {
- "name": "first-chunk",
- "repo": "repo",
- "ref": "original/ref"
- },
- {
- "name": "second-chunk",
- "repo": "repo",
- "ref": "original/ref"
- }
- ]
- }
- ''')
- first_stratum = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph,
- 'first-stratum.morph')
- pool.add(first_stratum)
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "second-stratum",
- "kind": "stratum",
- "build-depends": [
- "first-stratum"
- ],
- "sources": [
- {
- "name": "second-chunk",
- "repo": "repo",
- "ref": "original/ref"
- },
- {
- "name": "first-chunk",
- "repo": "repo",
- "ref": "original/ref"
- }
- ]
- }
- ''')
- second_stratum = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph,
- 'second-stratum.morph')
- pool.add(second_stratum)
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "first-chunk",
- "kind": "chunk"
- }
- ''')
- first_chunk = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph,
- 'first-chunk.morph')
- pool.add(first_chunk)
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "second-chunk",
- "kind": "chunk"
- }
- ''')
- second_chunk = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph,
- 'second-chunk.morph')
- pool.add(second_chunk)
-
- self.assertRaises(morphlib.buildgraph.MutualDependencyError,
- self.graph.compute_build_order, pool)
-
- def test_detection_of_cyclic_chunk_dependency_chain(self):
- pool = morphlib.sourcepool.SourcePool()
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "first-stratum",
- "kind": "stratum",
- "sources": [
- {
- "name": "first-chunk",
- "repo": "repo",
- "ref": "original/ref"
- },
- {
- "name": "second-chunk",
- "repo": "repo",
- "ref": "original/ref",
- "build-depends": [
- "first-chunk"
- ]
- },
- {
- "name": "third-chunk",
- "repo": "repo",
- "ref": "original/ref",
- "build-depends": [
- "second-chunk"
- ]
- }
- ]
- }
- ''')
- first_stratum = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph,
- 'first-stratum.morph')
- pool.add(first_stratum)
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "second-stratum",
- "kind": "stratum",
- "build-depends": [
- "first-stratum"
- ],
- "sources": [
- {
- "name": "third-chunk",
- "repo": "repo",
- "ref": "original/ref"
- },
- {
- "name": "first-chunk",
- "repo": "repo",
- "ref": "original/ref"
- }
- ]
- }
- ''')
- second_stratum = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph,
- 'second-stratum.morph')
- pool.add(second_stratum)
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "first-chunk",
- "kind": "chunk"
- }
- ''')
- first_chunk = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph,
- 'first-chunk.morph')
- pool.add(first_chunk)
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "second-chunk",
- "kind": "chunk"
- }
- ''')
- second_chunk = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph,
- 'second-chunk.morph')
- pool.add(second_chunk)
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "third-chunk",
- "kind": "chunk"
- }
- ''')
- second_chunk = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph,
- 'third-chunk.morph')
- pool.add(second_chunk)
-
- self.assertRaises(morphlib.buildgraph.CyclicDependencyChainError,
- self.graph.compute_build_order, pool)
-
- def test_detection_of_chunk_dependencies_in_invalid_order(self):
- pool = morphlib.sourcepool.SourcePool()
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "stratum",
- "kind": "stratum",
- "sources": [
- {
- "name": "first-chunk",
- "repo": "repo",
- "ref": "original/ref",
- "build-depends": [
- "second-chunk"
- ]
- },
- {
- "name": "second-chunk",
- "repo": "repo",
- "ref": "original/ref"
- }
- ]
- }
- ''')
- stratum = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph, 'stratum.morph')
- pool.add(stratum)
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "first-chunk",
- "kind": "chunk"
- }
- ''')
- first_chunk = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph, 'first-chunk.morph')
- pool.add(first_chunk)
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "second-chunk",
- "kind": "chunk"
- }
- ''')
- second_chunk = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph, 'second-chunk.morph')
- pool.add(second_chunk)
-
- self.assertRaises(morphlib.buildgraph.DependencyOrderError,
- self.graph.compute_build_order, pool)
-
- def test_detection_of_invalid_build_depends_format(self):
- pool = morphlib.sourcepool.SourcePool()
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "stratum",
- "kind": "stratum",
- "sources": [
- {
- "name": "chunk",
- "repo": "repo",
- "ref": "original/ref",
- "build-depends": "whatever"
- }
- ]
- }
- ''')
- stratum = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph, 'stratum.morph')
- pool.add(stratum)
-
- morph = morphlib.morph2.Morphology(
- '''
- {
- "name": "chunk",
- "kind": "chunk"
- }
- ''')
- chunk = morphlib.source.Source(
- 'repo', 'original/ref', 'sha1', morph, 'chunk.morph')
- pool.add(chunk)
-
- self.assertRaises(morphlib.buildgraph.DependencyFormatError,
- self.graph.compute_build_order, pool)
diff --git a/morphlib/buildorder_tests.py b/morphlib/buildorder_tests.py
index 67fbdf4f..32a14120 100644
--- a/morphlib/buildorder_tests.py
+++ b/morphlib/buildorder_tests.py
@@ -31,7 +31,7 @@ class BuildOrderTests(unittest.TestCase):
def test_list_with_one_artifact_results_in_one_group(self):
chunk = FakeSource()
- artifact = morphlib.artifact.Artifact(chunk, 'chunk', 'key')
+ artifact = morphlib.artifact.Artifact(chunk, 'chunk')
order = morphlib.buildorder.BuildOrder([artifact])
@@ -40,10 +40,10 @@ class BuildOrderTests(unittest.TestCase):
def test_list_with_two_unrelated_artifacts_results_in_one_group(self):
chunk1 = FakeSource()
- artifact1 = morphlib.artifact.Artifact(chunk1, 'chunk1', 'key1')
+ artifact1 = morphlib.artifact.Artifact(chunk1, 'chunk1')
chunk2 = FakeSource()
- artifact2 = morphlib.artifact.Artifact(chunk2, 'chunk2', 'key2')
+ artifact2 = morphlib.artifact.Artifact(chunk2, 'chunk2')
order = morphlib.buildorder.BuildOrder([artifact1, artifact2])
@@ -52,10 +52,10 @@ class BuildOrderTests(unittest.TestCase):
def test_list_with_two_dependent_artifacts_results_in_two_groups(self):
chunk1 = FakeSource()
- artifact1 = morphlib.artifact.Artifact(chunk1, 'chunk1', 'key1')
+ artifact1 = morphlib.artifact.Artifact(chunk1, 'chunk1')
chunk2 = FakeSource()
- artifact2 = morphlib.artifact.Artifact(chunk2, 'chunk2', 'key2')
+ artifact2 = morphlib.artifact.Artifact(chunk2, 'chunk2')
artifact2.add_dependency(artifact1)
order = morphlib.buildorder.BuildOrder([artifact1, artifact2])
@@ -66,14 +66,14 @@ class BuildOrderTests(unittest.TestCase):
def test_chain_of_three_dependent_artifacts_results_in_three_groups(self):
chunk1 = FakeSource()
- artifact1 = morphlib.artifact.Artifact(chunk1, 'chunk1', 'key1')
+ artifact1 = morphlib.artifact.Artifact(chunk1, 'chunk1')
chunk2 = FakeSource()
- artifact2 = morphlib.artifact.Artifact(chunk2, 'chunk2', 'key2')
+ artifact2 = morphlib.artifact.Artifact(chunk2, 'chunk2')
artifact2.add_dependency(artifact1)
chunk3 = FakeSource()
- artifact3 = morphlib.artifact.Artifact(chunk3, 'chunk3', 'key3')
+ artifact3 = morphlib.artifact.Artifact(chunk3, 'chunk3')
artifact3.add_dependency(artifact2)
order = morphlib.buildorder.BuildOrder(
@@ -86,14 +86,14 @@ class BuildOrderTests(unittest.TestCase):
def test_two_artifacts_depending_on_another_results_in_two_groups(self):
chunk1 = FakeSource()
- artifact1 = morphlib.artifact.Artifact(chunk1, 'chunk1', 'key1')
+ artifact1 = morphlib.artifact.Artifact(chunk1, 'chunk1')
chunk2 = FakeSource()
- artifact2 = morphlib.artifact.Artifact(chunk2, 'chunk2', 'key2')
+ artifact2 = morphlib.artifact.Artifact(chunk2, 'chunk2')
artifact2.add_dependency(artifact1)
chunk3 = FakeSource()
- artifact3 = morphlib.artifact.Artifact(chunk3, 'chunk3', 'key3')
+ artifact3 = morphlib.artifact.Artifact(chunk3, 'chunk3')
artifact3.add_dependency(artifact1)
order = morphlib.buildorder.BuildOrder(
@@ -105,13 +105,13 @@ class BuildOrderTests(unittest.TestCase):
def test_one_artifact_depending_on_two_others_results_in_two_groups(self):
chunk1 = FakeSource()
- artifact1 = morphlib.artifact.Artifact(chunk1, 'chunk1', 'key1')
+ artifact1 = morphlib.artifact.Artifact(chunk1, 'chunk1')
chunk2 = FakeSource()
- artifact2 = morphlib.artifact.Artifact(chunk2, 'chunk2', 'key2')
+ artifact2 = morphlib.artifact.Artifact(chunk2, 'chunk2')
chunk3 = FakeSource()
- artifact3 = morphlib.artifact.Artifact(chunk3, 'chunk3', 'key3')
+ artifact3 = morphlib.artifact.Artifact(chunk3, 'chunk3')
artifact3.add_dependency(artifact1)
artifact3.add_dependency(artifact2)
@@ -124,13 +124,13 @@ class BuildOrderTests(unittest.TestCase):
def test_detection_of_cyclic_dependency_chain(self):
chunk1 = FakeSource()
- artifact1 = morphlib.artifact.Artifact(chunk1, 'chunk1', 'key1')
+ artifact1 = morphlib.artifact.Artifact(chunk1, 'chunk1')
chunk2 = FakeSource()
- artifact2 = morphlib.artifact.Artifact(chunk2, 'chunk2', 'key2')
+ artifact2 = morphlib.artifact.Artifact(chunk2, 'chunk2')
chunk3 = FakeSource()
- artifact3 = morphlib.artifact.Artifact(chunk3, 'chunk3', 'key3')
+ artifact3 = morphlib.artifact.Artifact(chunk3, 'chunk3')
artifact1.add_dependency(artifact3)
artifact2.add_dependency(artifact1)
diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py
index 0a782185..28406b55 100644
--- a/morphlib/cachekeycomputer.py
+++ b/morphlib/cachekeycomputer.py
@@ -18,7 +18,8 @@ import hashlib
import morphlib
-class CacheKeyComputer():
+
+class CacheKeyComputer(object):
def __init__(self, build_env):
self._build_env = build_env
@@ -29,8 +30,8 @@ class CacheKeyComputer():
"TOOLCHAIN_TARGET", "PREFIX",
"BOOTSTRAP", "CFLAGS")])
- def compute_key(self, source):
- return self._hash_id(self.get_cache_id(source))
+ def compute_key(self, artifact):
+ return self._hash_id(self.get_cache_id(artifact))
def _hash_id(self, cache_id):
sha = hashlib.sha256()
@@ -59,20 +60,19 @@ class CacheKeyComputer():
for item in tup:
self._hash_thing(sha, item)
- def get_cache_id(self, source):
+ def get_cache_id(self, artifact):
try:
- return self._calculated[source]
+ return self._calculated[artifact]
except KeyError:
- cacheid = self._calculate(source)
- self._calculated[source] = cacheid
+ cacheid = self._calculate(artifact)
+ self._calculated[artifact] = cacheid
return cacheid
- def _calculate(self, source):
+ def _calculate(self, artifact):
return {
'arch': self._build_env.arch,
'env': self._filterenv(self._build_env.env),
- 'ref': source.sha1,
- 'filename': source.filename,
- 'kids': [self.get_cache_id(dependency)
- for dependency in source.dependencies],
+ 'ref': artifact.source.sha1,
+ 'filename': artifact.source.filename,
+ 'kids': [self.get_cache_id(x) for x in artifact.dependencies]
}
diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py
index 923cc9f9..51616a12 100644
--- a/morphlib/cachekeycomputer_tests.py
+++ b/morphlib/cachekeycomputer_tests.py
@@ -18,6 +18,7 @@ import unittest
import morphlib
+
class DummyBuildEnvironment:
'''Fake build environment class that doesn't need
settings to pick the environment, it just gets passed
@@ -27,11 +28,11 @@ class DummyBuildEnvironment:
self.arch = morphlib.util.arch() if arch == None else arch
self.env = env
+
class CacheKeyComputerTests(unittest.TestCase):
def setUp(self):
- pool = morphlib.sourcepool.SourcePool()
- self.sources = {}
+ self.source_pool = morphlib.sourcepool.SourcePool()
for name, text in {
'chunk.morph': '''{
"name": "chunk",
@@ -83,9 +84,7 @@ class CacheKeyComputerTests(unittest.TestCase):
}.iteritems():
source = morphlib.source.Source('repo', 'original/ref', 'sha',
morphlib.morph2.Morphology(text), name)
- pool.add(source)
- self.sources[name] = source
- morphlib.buildgraph.BuildGraph().compute_build_order(pool)
+ self.source_pool.add(source)
self.build_env = DummyBuildEnvironment({
"USER": "foouser",
"USERNAME": "foouser",
@@ -94,8 +93,17 @@ class CacheKeyComputerTests(unittest.TestCase):
"PREFIX": "/baserock",
"BOOTSTRAP": "false",
"CFLAGS": "-O4"})
+ self.artifact_resolver = morphlib.artifactresolver.ArtifactResolver()
+ self.artifacts = self.artifact_resolver.resolve_artifacts(
+ self.source_pool)
self.ckc = morphlib.cachekeycomputer.CacheKeyComputer(self.build_env)
+ def _find_artifact(self, name):
+ for artifact in self.artifacts:
+ if artifact.name == name:
+ return artifact
+ raise
+
def test_compute_key_hashes_all_types(self):
runcount = {'thing': 0, 'dict': 0, 'list': 0, 'tuple': 0}
def inccount(func, name):
@@ -103,11 +111,15 @@ class CacheKeyComputerTests(unittest.TestCase):
runcount[name] = runcount[name] + 1
func(sha, item)
return f
+
self.ckc._hash_thing = inccount(self.ckc._hash_thing, 'thing')
self.ckc._hash_dict = inccount(self.ckc._hash_dict, 'dict')
self.ckc._hash_list = inccount(self.ckc._hash_list, 'list')
self.ckc._hash_tuple = inccount(self.ckc._hash_tuple, 'tuple')
- self.ckc.compute_key(self.sources['stratum.morph'])
+
+ artifact = self._find_artifact('system')
+ self.ckc.compute_key(artifact)
+
self.assertNotEqual(runcount['thing'], 0)
self.assertNotEqual(runcount['dict'], 0)
self.assertNotEqual(runcount['list'], 0)
@@ -115,14 +127,16 @@ class CacheKeyComputerTests(unittest.TestCase):
def _valid_sha256(self, s):
validchars = '0123456789abcdef'
- return len(s) == 64 and all(c in validchars for c in s)
+ return len(s) == 64 and all([c in validchars for c in s])
def test_compute_key_returns_sha256(self):
+ artifact = self._find_artifact('system')
self.assertTrue(self._valid_sha256(
- self.ckc.compute_key(self.sources['system.morph'])))
+ self.ckc.compute_key(artifact)))
def test_different_env_gives_different_key(self):
- oldsha = self.ckc.compute_key(self.sources['system.morph'])
+ artifact = self._find_artifact('system')
+ oldsha = self.ckc.compute_key(artifact)
build_env = DummyBuildEnvironment({
"USER": "foouser",
"USERNAME": "foouser",
@@ -133,5 +147,4 @@ class CacheKeyComputerTests(unittest.TestCase):
"CFLAGS": "-Os"})
ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env)
- self.assertNotEqual(oldsha,
- ckc.compute_key(self.sources['system.morph']))
+ self.assertNotEqual(oldsha, ckc.compute_key(artifact))
diff --git a/morphlib/localartifactcache.py b/morphlib/localartifactcache.py
index 52b5772c..cf025210 100644
--- a/morphlib/localartifactcache.py
+++ b/morphlib/localartifactcache.py
@@ -60,17 +60,12 @@ class LocalArtifactCache(object):
filename = self._source_metadata_filename(source, cachekey, name)
return open(filename)
- def _artifact_basename(self, artifact):
- return '%s.%s.%s' % (artifact.cache_key,
- artifact.source.morphology['kind'],
- artifact.name)
-
def _artifact_filename(self, artifact):
- basename = self._artifact_basename(artifact)
+ basename = artifact.basename()
return os.path.join(self.cachedir, basename)
def _artifact_metadata_filename(self, artifact, name):
- basename = '%s.%s' % (self._artifact_basename(artifact), name)
+ basename = '%s.%s' % (artifact.basename(), name)
return os.path.join(self.cachedir, basename)
diff --git a/morphlib/localartifactcache_tests.py b/morphlib/localartifactcache_tests.py
index 41eb3d0b..a2fb0810 100644
--- a/morphlib/localartifactcache_tests.py
+++ b/morphlib/localartifactcache_tests.py
@@ -45,9 +45,9 @@ class LocalArtifactCacheTests(unittest.TestCase):
self.source = morphlib.source.Source(
'repo', 'ref', 'sha1', morph, 'chunk.morph')
self.runtime_artifact = morphlib.artifact.Artifact(
- self.source, 'chunk-runtime', 'cachekey')
+ self.source, 'chunk-runtime')
self.devel_artifact = morphlib.artifact.Artifact(
- self.source, 'chunk-devel', 'cachekey')
+ self.source, 'chunk-devel')
def tearDown(self):
self.tempdir.remove()