diff options
Diffstat (limited to 'morphlib')
-rw-r--r-- | morphlib/__init__.py | 3 | ||||
-rw-r--r-- | morphlib/artifact_tests.py | 4 | ||||
-rw-r--r-- | morphlib/artifactresolver.py | 204 | ||||
-rw-r--r-- | morphlib/artifactresolver_tests.py | 589 | ||||
-rw-r--r-- | morphlib/artifactsplitrule.py | 303 | ||||
-rw-r--r-- | morphlib/bins.py | 70 | ||||
-rw-r--r-- | morphlib/bins_tests.py | 39 | ||||
-rw-r--r-- | morphlib/buildcommand.py | 32 | ||||
-rw-r--r-- | morphlib/builder2.py | 84 | ||||
-rw-r--r-- | morphlib/cachekeycomputer.py | 4 | ||||
-rw-r--r-- | morphlib/cachekeycomputer_tests.py | 16 | ||||
-rw-r--r-- | morphlib/localartifactcache_tests.py | 4 | ||||
-rw-r--r-- | morphlib/morph2.py | 13 | ||||
-rw-r--r-- | morphlib/morph2_tests.py | 6 | ||||
-rw-r--r-- | morphlib/morphloader.py | 120 | ||||
-rw-r--r-- | morphlib/morphloader_tests.py | 98 | ||||
-rw-r--r-- | morphlib/morphologyfactory.py | 9 | ||||
-rw-r--r-- | morphlib/morphologyfactory_tests.py | 16 | ||||
-rw-r--r-- | morphlib/remoteartifactcache_tests.py | 4 | ||||
-rw-r--r-- | morphlib/source.py | 11 |
20 files changed, 923 insertions, 706 deletions
diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 67fb944d..33773791 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -48,6 +48,7 @@ class Error(cliapp.AppException): import artifact import artifactcachereference import artifactresolver +import artifactsplitrule import branchmanager import bins import buildbranch diff --git a/morphlib/artifact_tests.py b/morphlib/artifact_tests.py index 8edbbde2..d4b15cba 100644 --- a/morphlib/artifact_tests.py +++ b/morphlib/artifact_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -26,7 +26,7 @@ class ArtifactTests(unittest.TestCase): morph = morphlib.morph2.Morphology( ''' { - "chunk": "chunk", + "name": "chunk", "kind": "chunk", "chunks": { "chunk-runtime": [ diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index 17f038a2..ae0cfcf5 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,6 +16,7 @@ import cliapp import collections +import logging import morphlib @@ -29,35 +30,21 @@ class MutualDependencyError(cliapp.AppException): class DependencyOrderError(cliapp.AppException): - def __init__(self, stratum, chunk, dependency_name): + def __init__(self, stratum_source, chunk, dependency_name): cliapp.AppException.__init__( self, 'In stratum %s, chunk %s references its dependency %s ' 'before it is defined' % - (stratum.source, chunk, dependency_name)) + (stratum_source, chunk, dependency_name)) class DependencyFormatError(cliapp.AppException): - def __init__(self, stratum, chunk): + def __init__(self, stratum_source, chunk): cliapp.AppException.__init__( self, 'In stratum %s, chunk %s uses an invalid ' - 'build-depends format' % (stratum.source, chunk)) + 'build-depends format' % (stratum_source, chunk)) -class UndefinedChunkArtifactError(cliapp.AppException): - - '''Exception raised when non-existent artifacts are referenced. - - Usually, this will only occur when a stratum refers to a chunk - artifact that is not defined in a chunk. - - ''' - - def __init__(self, parent, reference): - cliapp.AppException.__init__( - self, 'Undefined chunk artifact "%s" referenced in ' - 'stratum %s' % (reference, parent)) - class ArtifactResolver(object): @@ -71,13 +58,11 @@ class ArtifactResolver(object): ''' def __init__(self): - self._cached_artifacts = None self._added_artifacts = None self._source_pool = None def resolve_artifacts(self, source_pool): self._source_pool = source_pool - self._cached_artifacts = {} self._added_artifacts = set() artifacts = self._resolve_artifacts_recursively() @@ -93,12 +78,13 @@ class ArtifactResolver(object): source = queue.popleft() if source.morphology['kind'] == 'system': - systems = [self._get_artifact(source, a) - for a in source.morphology.builds_artifacts] + systems = [source.artifacts[name] + for name in source.split_rules.artifacts] - if any(a not in self._added_artifacts for a in systems): - artifacts.extend(systems) - self._added_artifacts.update(systems) + for system in (s for s in systems + if s not in self._added_artifacts): + artifacts.append(system) + self._added_artifacts.add(system) resolved_artifacts = self._resolve_system_dependencies( systems, source, queue) @@ -108,28 +94,36 @@ class ArtifactResolver(object): artifacts.append(artifact) self._added_artifacts.add(artifact) elif source.morphology['kind'] == 'stratum': - assert len(source.morphology.builds_artifacts) == 1 - artifact = self._get_artifact( - source, source.morphology.builds_artifacts[0]) - - if not artifact in self._added_artifacts: - artifacts.append(artifact) - self._added_artifacts.add(artifact) + strata = [source.artifacts[name] + for name in source.split_rules.artifacts] + + # If we were not given systems, return the strata here, + # rather than have the systems return them. + if not any(s.morphology['kind'] == 'system' + for s in self._source_pool): + for stratum in (s for s in strata + if s not in self._added_artifacts): + artifacts.append(stratum) + self._added_artifacts.add(stratum) resolved_artifacts = self._resolve_stratum_dependencies( - artifact, queue) + strata, source, queue) for artifact in resolved_artifacts: if not artifact in self._added_artifacts: artifacts.append(artifact) self._added_artifacts.add(artifact) elif source.morphology['kind'] == 'chunk': - names = source.morphology.builds_artifacts - for name in names: - artifact = self._get_artifact(source, name) - if not artifact in self._added_artifacts: - artifacts.append(artifact) - self._added_artifacts.add(artifact) + chunks = [source.artifacts[name] + for name in source.split_rules.artifacts] + # If we were only given chunks, return them here, rather than + # have the strata return them. + if not any(s.morphology['kind'] == 'stratum' + for s in self._source_pool): + for chunk in (c for c in chunks + if c not in self._added_artifacts): + artifacts.append(chunk) + self._added_artifacts.add(chunk) return artifacts @@ -141,15 +135,6 @@ class ArtifactResolver(object): if x.morphology['kind'] != 'chunk'] return collections.deque(sources) - def _get_artifact(self, source, name): - info = (source, name) - if info in self._cached_artifacts: - return self._cached_artifacts[info] - else: - artifact = morphlib.artifact.Artifact(info[0], info[1]) - self._cached_artifacts[info] = artifact - return artifact - def _resolve_system_dependencies(self, systems, source, queue): artifacts = [] @@ -158,67 +143,60 @@ class ArtifactResolver(object): info['repo'] or source.repo_name, info['ref'] or source.original_ref, '%s.morph' % info['morph']) + stratum_name = stratum_source.morphology['name'] - stratum_name = stratum_source.morphology.builds_artifacts[0] - stratum = self._get_artifact(stratum_source, stratum_name) - + matches, overlaps, unmatched = source.split_rules.partition( + ((stratum_name, sta_name) for sta_name + in stratum_source.split_rules.artifacts)) for system in systems: - system.add_dependency(stratum) - queue.append(stratum_source) + for (stratum_name, sta_name) in matches[system.name]: + stratum = stratum_source.artifacts[sta_name] + system.add_dependency(stratum) + artifacts.append(stratum) - artifacts.append(stratum) + queue.append(stratum_source) return artifacts - def _resolve_stratum_dependencies(self, stratum, queue): + def _resolve_stratum_dependencies(self, strata, source, queue): artifacts = [] - strata = [] + stratum_build_depends = [] - if stratum.source.morphology['build-depends']: - for stratum_info in stratum.source.morphology['build-depends']: - other_source = self._source_pool.lookup( - stratum_info['repo'] or stratum.source.repo_name, - stratum_info['ref'] or stratum.source.original_ref, - '%s.morph' % stratum_info['morph']) + for stratum_info in source.morphology.get('build-depends') or []: + other_source = self._source_pool.lookup( + stratum_info['repo'] or source.repo_name, + stratum_info['ref'] or source.original_ref, + '%s.morph' % stratum_info['morph']) - other_stratum = self._get_artifact( - other_source, other_source.morphology.builds_artifacts[0]) + # Make every stratum artifact this stratum source produces + # depend on every stratum artifact the other stratum source + # produces. + for sta_name in other_source.split_rules.artifacts: + other_stratum = other_source.artifacts[sta_name] - strata.append(other_stratum) + stratum_build_depends.append(other_stratum) artifacts.append(other_stratum) - if other_stratum.depends_on(stratum): - raise MutualDependencyError(stratum, other_stratum) + for stratum in strata: + if other_stratum.depends_on(stratum): + raise MutualDependencyError(stratum, other_stratum) + + stratum.add_dependency(other_stratum) - stratum.add_dependency(other_stratum) - queue.append(other_source) + queue.append(other_source) # 'name' here is the chunk artifact name - chunk_artifacts = [] - processed_artifacts = [] - name_to_processed_artifact = {} + name_to_processed_artifacts = {} - for info in stratum.source.morphology['chunks']: + for info in source.morphology['chunks']: chunk_source = self._source_pool.lookup( info['repo'], info['ref'], '%s.morph' % info['morph']) - possible_names = chunk_source.morphology.builds_artifacts - if not info['name'] in possible_names: - raise UndefinedChunkArtifactError(stratum.source, info['name']) - - chunk_artifact = self._get_artifact(chunk_source, info['name']) - chunk_artifacts.append(chunk_artifact) - - artifacts.append(chunk_artifact) - - stratum.add_dependency(chunk_artifact) - - for other_stratum in strata: - chunk_artifact.add_dependency(other_stratum) + chunk_name = chunk_source.morphology['name'] # Resolve now to avoid a search for the parent morphology later chunk_source.build_mode = info['build-mode'] @@ -226,23 +204,45 @@ class ArtifactResolver(object): build_depends = info.get('build-depends', None) - if build_depends is None: - for earlier_artifact in processed_artifacts: - if earlier_artifact.depends_on(chunk_artifact): - raise MutualDependencyError( - chunk_artifact, earlier_artifact) - chunk_artifact.add_dependency(earlier_artifact) - elif isinstance(build_depends, list): + for ca_name in chunk_source.split_rules.artifacts: + chunk_artifact = chunk_source.artifacts[ca_name] + + # Add our stratum's build depends as dependencies of this chunk + for other_stratum in stratum_build_depends: + chunk_artifact.add_dependency(other_stratum) + + # Add dependencies between chunks mentioned in this stratum + if isinstance(build_depends, list): for name in build_depends: - other_artifact = name_to_processed_artifact.get(name, None) - if other_artifact: - chunk_artifact.add_dependency(other_artifact) - else: + if name not in name_to_processed_artifacts: raise DependencyOrderError( - stratum, info['name'], name) + source, info['name'], name) + other_artifacts = name_to_processed_artifacts[name] + for other_artifact in other_artifacts: + for ca_name in chunk_source.split_rules.artifacts: + chunk_artifact = chunk_source.artifacts[ca_name] + chunk_artifact.add_dependency(other_artifact) else: - raise DependencyFormatError(stratum, info['name']) - processed_artifacts.append(chunk_artifact) - name_to_processed_artifact[info['name']] = chunk_artifact + raise DependencyFormatError(source, info['name']) + + # Add build dependencies between our stratum's artifacts + # and the chunk artifacts produced by this stratum. + matches, overlaps, unmatched = source.split_rules.partition( + ((chunk_name, ca_name) for ca_name + in chunk_source.split_rules.artifacts)) + for stratum in strata: + for (chunk_name, ca_name) in matches[stratum.name]: + chunk_artifact = chunk_source.artifacts[ca_name] + stratum.add_dependency(chunk_artifact) + # Only return chunks required to build strata we need + if chunk_artifact not in artifacts: + artifacts.append(chunk_artifact) + + + # Add these chunks to the processed artifacts, so other + # chunks may refer to them. + name_to_processed_artifacts[info['name']] = \ + [chunk_source.artifacts[n] for n + in chunk_source.split_rules.artifacts] return artifacts diff --git a/morphlib/artifactresolver_tests.py b/morphlib/artifactresolver_tests.py index b685902e..6f62b4d1 100644 --- a/morphlib/artifactresolver_tests.py +++ b/morphlib/artifactresolver_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,6 +14,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import itertools import json import unittest @@ -27,16 +28,15 @@ class FakeChunkMorphology(morphlib.morph2.Morphology): if artifact_names: # fake a list of artifacts - artifacts = {} + artifacts = [] for artifact_name in artifact_names: - artifacts[artifact_name] = [artifact_name] - text = (''' - { - "name": "%s", + artifacts.append({'artifact': artifact_name, + 'include': artifact_name}) + text = json.dumps({ + "name": name, "kind": "chunk", - "chunks": %s - } - ''' % (name, json.dumps(artifacts))) + "products": artifacts + }) self.builds_artifacts = artifact_names else: text = (''' @@ -61,7 +61,8 @@ class FakeStratumMorphology(morphlib.morph2.Morphology): 'name': source_name, 'morph': morph, 'repo': repo, - 'ref': ref + 'ref': ref, + 'build-depends': [], }) build_depends_list = [] for morph, repo, ref in build_depends: @@ -114,33 +115,37 @@ class ArtifactResolverTests(unittest.TestCase): artifacts = self.resolver.resolve_artifacts(pool) - self.assertEqual(len(artifacts), 1) + self.assertEqual(len(artifacts), + sum(len(s.split_rules.artifacts) for s in pool)) - self.assertEqual(artifacts[0].source, source) - self.assertEqual(artifacts[0].name, 'chunk') - self.assertEqual(artifacts[0].dependencies, []) - self.assertEqual(artifacts[0].dependents, []) + for artifact in artifacts: + self.assertEqual(artifact.source, source) + self.assertTrue(artifact.name.startswith('chunk')) + self.assertEqual(artifact.dependencies, []) + self.assertEqual(artifact.dependents, []) - def test_resolve_single_chunk_with_one_artifact(self): + def test_resolve_single_chunk_with_one_new_artifact(self): pool = morphlib.sourcepool.SourcePool() - morph = FakeChunkMorphology('chunk', ['chunk-runtime']) + morph = FakeChunkMorphology('chunk', ['chunk-foobar']) source = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') pool.add(source) artifacts = self.resolver.resolve_artifacts(pool) - self.assertEqual(len(artifacts), 1) - self.assertEqual(artifacts[0].source, source) - self.assertEqual(artifacts[0].name, 'chunk-runtime') - self.assertEqual(artifacts[0].dependencies, []) - self.assertEqual(artifacts[0].dependents, []) + self.assertEqual(len(artifacts), + sum(len(s.split_rules.artifacts) for s in pool)) + + foobartifact, = (a for a in artifacts if a.name == 'chunk-foobar') + self.assertEqual(foobartifact.source, source) + self.assertEqual(foobartifact.dependencies, []) + self.assertEqual(foobartifact.dependents, []) - def test_resolve_single_chunk_with_two_artifact(self): + def test_resolve_single_chunk_with_two_new_artifacts(self): pool = morphlib.sourcepool.SourcePool() - morph = FakeChunkMorphology('chunk', ['chunk-runtime', 'chunk-devel']) + morph = FakeChunkMorphology('chunk', ['chunk-baz', 'chunk-qux']) source = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') pool.add(source) @@ -148,88 +153,16 @@ class ArtifactResolverTests(unittest.TestCase): artifacts = self.resolver.resolve_artifacts(pool) artifacts.sort(key=lambda a: a.name) - self.assertEqual(len(artifacts), 2) + self.assertEqual(len(artifacts), + sum(len(s.split_rules.artifacts) for s in pool)) - self.assertEqual(artifacts[0].source, source) - self.assertEqual(artifacts[0].name, 'chunk-devel') - self.assertEqual(artifacts[0].dependencies, []) - self.assertEqual(artifacts[0].dependents, []) + for name in ('chunk-baz', 'chunk-qux'): + artifact, = (a for a in artifacts if a.name == name) + self.assertEqual(artifact.source, source) + self.assertEqual(artifact.dependencies, []) + self.assertEqual(artifact.dependents, []) - self.assertEqual(artifacts[1].source, source) - self.assertEqual(artifacts[1].name, 'chunk-runtime') - self.assertEqual(artifacts[1].dependencies, []) - self.assertEqual(artifacts[1].dependents, []) - - def test_resolve_a_single_empty_stratum(self): - pool = morphlib.sourcepool.SourcePool() - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "foo", - "kind": "stratum" - } - ''') - morph.builds_artifacts = ['foo'] - stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'foo.morph') - pool.add(stratum) - - artifacts = self.resolver.resolve_artifacts(pool) - - self.assertEqual(artifacts[0].source, stratum) - self.assertEqual(artifacts[0].name, 'foo') - self.assertEqual(artifacts[0].dependencies, []) - self.assertEqual(artifacts[0].dependents, []) - - def test_resolve_a_single_empty_system(self): - pool = morphlib.sourcepool.SourcePool() - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "foo", - "kind": "system" - } - ''') - morph.builds_artifacts = ['foo-rootfs'] - system = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'foo.morph') - pool.add(system) - - artifacts = self.resolver.resolve_artifacts(pool) - - self.assertEqual(artifacts[0].source, system) - self.assertEqual(artifacts[0].name, 'foo-rootfs') - self.assertEqual(artifacts[0].dependencies, []) - self.assertEqual(artifacts[0].dependents, []) - - def test_resolve_a_single_empty_arm_system(self): - pool = morphlib.sourcepool.SourcePool() - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "foo", - "kind": "system", - "arch": "armv7" - } - ''') - morph.builds_artifacts = ['foo-rootfs', 'foo-kernel'] - system = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'foo.morph') - pool.add(system) - - artifacts = self.resolver.resolve_artifacts(pool) - - self.assertTrue(any((a.source == system and a.name == 'foo-rootfs' and - a.dependencies == [] and a.dependents == []) - for a in artifacts)) - self.assertTrue(any((a.source == system and a.name == 'foo-kernel' and - a.dependencies == [] and a.dependents == []) - for a in artifacts)) - - def test_resolve_stratum_and_chunk_with_no_subartifacts(self): + def test_resolve_stratum_and_chunk(self): pool = morphlib.sourcepool.SourcePool() morph = FakeChunkMorphology('chunk') @@ -245,103 +178,36 @@ class ArtifactResolverTests(unittest.TestCase): artifacts = self.resolver.resolve_artifacts(pool) - self.assertEqual(len(artifacts), 2) + self.assertEqual(len(artifacts), + sum(len(s.split_rules.artifacts) for s in pool)) - self.assertEqual(artifacts[0].source, stratum) - self.assertEqual(artifacts[0].name, 'stratum') - self.assertEqual(artifacts[0].dependencies, [artifacts[1]]) - self.assertEqual(artifacts[0].dependents, []) + stratum_artifacts = set(a for a in artifacts if a.source == stratum) + chunk_artifacts = set(a for a in artifacts if a.source == chunk) - self.assertEqual(artifacts[1].source, chunk) - self.assertEqual(artifacts[1].name, 'chunk') - self.assertEqual(artifacts[1].dependencies, []) - self.assertEqual(artifacts[1].dependents, [artifacts[0]]) + for stratum_artifact in stratum_artifacts: + self.assertTrue(stratum_artifact.name.startswith('stratum')) + self.assertEqual(stratum_artifact.dependents, []) + self.assertTrue(any(dep in chunk_artifacts + for dep in stratum_artifact.dependencies)) - def test_resolve_stratum_and_chunk_with_two_subartifacts(self): - pool = morphlib.sourcepool.SourcePool() + for chunk_artifact in chunk_artifacts: + self.assertTrue(chunk_artifact.name.startswith('chunk')) + self.assertEqual(chunk_artifact.dependencies, []) + self.assertTrue(any(dep in stratum_artifacts + for dep in chunk_artifact.dependents)) - morph = FakeChunkMorphology('chunk', ['chunk-devel', 'chunk-runtime']) - chunk = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') - pool.add(chunk) - - morph = FakeStratumMorphology( - 'stratum', - chunks=[ - ('chunk-devel', 'chunk', 'repo', 'ref'), - ('chunk-runtime', 'chunk', 'repo', 'ref') - ]) - stratum = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'stratum.morph') - pool.add(stratum) - - artifacts = self.resolver.resolve_artifacts(pool) - - self.assertEqual(len(artifacts), 3) - - self.assertEqual(artifacts[0].source, stratum) - self.assertEqual(artifacts[0].name, 'stratum') - self.assertEqual(artifacts[0].dependencies, - [artifacts[1], artifacts[2]]) - self.assertEqual(artifacts[0].dependents, []) - - self.assertEqual(artifacts[1].source, chunk) - self.assertEqual(artifacts[1].name, 'chunk-devel') - self.assertEqual(artifacts[1].dependencies, []) - self.assertEqual(artifacts[1].dependents, [artifacts[0], artifacts[2]]) - - self.assertEqual(artifacts[2].source, chunk) - self.assertEqual(artifacts[2].name, 'chunk-runtime') - self.assertEqual(artifacts[2].dependencies, [artifacts[1]]) - self.assertEqual(artifacts[2].dependents, [artifacts[0]]) - - def test_resolve_stratum_and_chunk_with_one_used_subartifacts(self): + def test_resolve_stratum_and_chunk_with_two_new_artifacts(self): pool = morphlib.sourcepool.SourcePool() - morph = FakeChunkMorphology('chunk', ['chunk-devel', 'chunk-runtime']) + morph = FakeChunkMorphology('chunk', ['chunk-foo', 'chunk-bar']) chunk = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') pool.add(chunk) morph = FakeStratumMorphology( 'stratum', - chunks=[('chunk-runtime', 'chunk', 'repo', 'ref')]) - stratum = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'stratum.morph') - pool.add(stratum) - - artifacts = self.resolver.resolve_artifacts(pool) - - self.assertEqual(len(artifacts), 2) - - self.assertEqual(artifacts[0].source, stratum) - self.assertEqual(artifacts[0].name, 'stratum') - self.assertEqual(artifacts[0].dependencies, [artifacts[1]]) - self.assertEqual(artifacts[0].dependents, []) - - self.assertEqual(artifacts[1].source, chunk) - self.assertEqual(artifacts[1].name, 'chunk-runtime') - self.assertEqual(artifacts[1].dependencies, []) - self.assertEqual(artifacts[1].dependents, [artifacts[0]]) - - def test_resolving_two_different_chunk_artifacts_in_a_stratum(self): - pool = morphlib.sourcepool.SourcePool() - - morph = FakeChunkMorphology('foo') - foo_chunk = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'foo.morph') - pool.add(foo_chunk) - - morph = FakeChunkMorphology('bar') - bar_chunk = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'bar.morph') - pool.add(bar_chunk) - - morph = FakeStratumMorphology( - 'stratum', chunks=[ - ('foo', 'foo', 'repo', 'ref'), - ('bar', 'bar', 'repo', 'ref') + ('chunk', 'chunk', 'repo', 'ref'), ]) stratum = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'stratum.morph') @@ -349,114 +215,35 @@ class ArtifactResolverTests(unittest.TestCase): artifacts = self.resolver.resolve_artifacts(pool) - self.assertEqual(len(artifacts), 3) - - self.assertEqual(artifacts[0].source, stratum) - self.assertEqual(artifacts[0].name, 'stratum') - self.assertEqual(artifacts[0].dependencies, - [artifacts[1], artifacts[2]]) - self.assertEqual(artifacts[0].dependents, []) - - self.assertEqual(artifacts[1].source, foo_chunk) - self.assertEqual(artifacts[1].name, 'foo') - self.assertEqual(artifacts[1].dependencies, []) - self.assertEqual(artifacts[1].dependents, [artifacts[0], artifacts[2]]) - - self.assertEqual(artifacts[2].source, bar_chunk) - self.assertEqual(artifacts[2].name, 'bar') - self.assertEqual(artifacts[2].dependencies, [artifacts[1]]) - self.assertEqual(artifacts[2].dependents, [artifacts[0]]) - - def test_resolving_artifacts_for_a_chain_of_two_strata(self): - pool = morphlib.sourcepool.SourcePool() - - morph = FakeStratumMorphology('stratum1') - stratum1 = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'stratum1.morph') - pool.add(stratum1) - - morph = FakeStratumMorphology( - 'stratum2', - chunks=[], - build_depends=[('stratum1', 'repo', 'ref')]) - stratum2 = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'stratum2.morph') - pool.add(stratum2) - - artifacts = self.resolver.resolve_artifacts(pool) + self.assertEqual(len(artifacts), + sum(len(s.split_rules.artifacts) for s in pool)) - self.assertEqual(len(artifacts), 2) + stratum_artifacts = set(a for a in artifacts if a.source == stratum) + chunk_artifacts = set(a for a in artifacts if a.source == chunk) - self.assertEqual(artifacts[0].source, stratum1) - self.assertEqual(artifacts[0].name, 'stratum1') - self.assertEqual(artifacts[0].dependencies, []) - self.assertEqual(artifacts[0].dependents, [artifacts[1]]) + for stratum_artifact in stratum_artifacts: + self.assertTrue(stratum_artifact.name.startswith('stratum')) + self.assertEqual(stratum_artifact.dependents, []) + self.assertTrue(any(dep in chunk_artifacts + for dep in stratum_artifact.dependencies)) - self.assertEqual(artifacts[1].source, stratum2) - self.assertEqual(artifacts[1].name, 'stratum2') - self.assertEqual(artifacts[1].dependencies, [artifacts[0]]) - self.assertEqual(artifacts[1].dependents, []) + for chunk_artifact in chunk_artifacts: + self.assertTrue(chunk_artifact.name.startswith('chunk')) + self.assertEqual(chunk_artifact.dependencies, []) + self.assertTrue(any(dep in stratum_artifacts + for dep in chunk_artifact.dependents)) - def test_resolving_with_a_stratum_and_chunk_dependency_mix(self): + def test_resolving_artifacts_for_a_system_with_two_dependent_strata(self): pool = morphlib.sourcepool.SourcePool() - morph = FakeStratumMorphology('stratum1') - stratum1 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'stratum1.morph') - pool.add(stratum1) - - morph = FakeStratumMorphology( - 'stratum2', - chunks=[ - ('chunk1', 'chunk1', 'repo', 'original/ref'), - ('chunk2', 'chunk2', 'repo', 'original/ref') - ], - build_depends=[('stratum1', 'repo', 'original/ref')]) - stratum2 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'stratum2.morph') - pool.add(stratum2) - morph = FakeChunkMorphology('chunk1') chunk1 = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk1.morph') pool.add(chunk1) - morph = FakeChunkMorphology('chunk2') - chunk2 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk2.morph') - pool.add(chunk2) - - artifacts = self.resolver.resolve_artifacts(pool) - - self.assertEqual(len(artifacts), 4) - - self.assertEqual(artifacts[0].source, stratum1) - self.assertEqual(artifacts[0].name, 'stratum1') - self.assertEqual(artifacts[0].dependencies, []) - self.assertEqual(artifacts[0].dependents, - [artifacts[1], artifacts[2], artifacts[3]]) - - self.assertEqual(artifacts[1].source, stratum2) - self.assertEqual(artifacts[1].name, 'stratum2') - self.assertEqual(artifacts[1].dependencies, - [artifacts[0], artifacts[2], artifacts[3]]) - self.assertEqual(artifacts[1].dependents, []) - - self.assertEqual(artifacts[2].source, chunk1) - self.assertEqual(artifacts[2].name, 'chunk1') - self.assertEqual(artifacts[2].dependencies, [artifacts[0]]) - self.assertEqual(artifacts[2].dependents, [artifacts[1], artifacts[3]]) - - self.assertEqual(artifacts[3].source, chunk2) - self.assertEqual(artifacts[3].name, 'chunk2') - self.assertEqual(artifacts[3].dependencies, - [artifacts[0], artifacts[2]]) - self.assertEqual(artifacts[3].dependents, [artifacts[1]]) - - def test_resolving_artifacts_for_a_system_with_two_strata(self): - pool = morphlib.sourcepool.SourcePool() - - morph = FakeStratumMorphology('stratum1') + morph = FakeStratumMorphology( + 'stratum1', + chunks=[('chunk1', 'chunk1', 'repo', 'original/ref')]) stratum1 = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'stratum1.morph') pool.add(stratum1) @@ -485,31 +272,67 @@ class ArtifactResolverTests(unittest.TestCase): 'repo', 'ref', 'sha1', 'tree', morph, 'system.morph') pool.add(system) + morph = FakeChunkMorphology('chunk2') + chunk2 = morphlib.source.Source( + 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk2.morph') + pool.add(chunk2) + morph = FakeStratumMorphology( - 'stratum2', chunks=[], build_depends=[('stratum1', 'repo', 'ref')]) + 'stratum2', + chunks=[('chunk2', 'chunk2', 'repo', 'original/ref')], + build_depends=[('stratum1', 'repo', 'ref')]) stratum2 = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'stratum2.morph') pool.add(stratum2) artifacts = self.resolver.resolve_artifacts(pool) - self.assertEqual(len(artifacts), 3) + self.assertEqual(len(artifacts), + sum(len(s.split_rules.artifacts) for s in pool)) + + system_artifacts = set(a for a in artifacts if a.source == system) + stratum1_artifacts = set(a for a in artifacts if a.source == stratum1) + chunk1_artifacts = set(a for a in artifacts if a.source == chunk1) + stratum2_artifacts = set(a for a in artifacts if a.source == stratum2) + chunk2_artifacts = set(a for a in artifacts if a.source == chunk2) + + def assert_depended_on_by_some(artifact, parents): + self.assertNotEqual(len(artifact.dependents), 0) + self.assertTrue(any(a in artifact.dependents for a in parents)) + + def assert_depended_on_by_all(artifact, parents): + self.assertNotEqual(len(artifact.dependents), 0) + self.assertTrue(all(a in artifact.dependents for a in parents)) + + def assert_depends_on_some(artifact, children): + self.assertNotEqual(len(artifact.dependencies), 0) + self.assertTrue(any(a in children for a in artifact.dependencies)) + + def assert_depends_on_all(artifact, children): + self.assertNotEqual(len(artifact.dependencies), 0) + self.assertTrue(all(a in children for a in artifact.dependencies)) - self.assertEqual(artifacts[0].source, stratum1) - self.assertEqual(artifacts[0].name, 'stratum1') - self.assertEqual(artifacts[0].dependencies, []) - self.assertEqual(artifacts[0].dependents, [artifacts[1], artifacts[2]]) + for c1_a in chunk1_artifacts: + self.assertEqual(c1_a.dependencies, []) + assert_depended_on_by_some(c1_a, stratum1_artifacts) - self.assertEqual(artifacts[1].source, system) - self.assertEqual(artifacts[1].name, 'system-rootfs') - self.assertEqual(artifacts[1].dependencies, - [artifacts[0], artifacts[2]]) - self.assertEqual(artifacts[1].dependents, []) + for st1_a in stratum1_artifacts: + assert_depends_on_some(st1_a, chunk1_artifacts) + assert_depended_on_by_all(st1_a, chunk2_artifacts) + assert_depended_on_by_some(st1_a, system_artifacts) - self.assertEqual(artifacts[2].source, stratum2) - self.assertEqual(artifacts[2].name, 'stratum2') - self.assertEqual(artifacts[2].dependencies, [artifacts[0]]) - self.assertEqual(artifacts[2].dependents, [artifacts[1]]) + for c2_a in chunk2_artifacts: + assert_depends_on_all(c2_a, stratum1_artifacts) + assert_depended_on_by_some(c2_a, stratum2_artifacts) + + for st2_a in stratum2_artifacts: + assert_depends_on_some(st2_a, chunk2_artifacts) + assert_depended_on_by_some(st2_a, system_artifacts) + + for sy_a in system_artifacts: + self.assertEqual(sy_a.dependents, []) + assert_depends_on_some(sy_a, stratum1_artifacts) + assert_depends_on_some(sy_a, stratum2_artifacts) def test_resolving_stratum_with_explicit_chunk_dependencies(self): pool = morphlib.sourcepool.SourcePool() @@ -566,49 +389,33 @@ class ArtifactResolverTests(unittest.TestCase): artifacts = self.resolver.resolve_artifacts(pool) - self.assertEqual(len(artifacts), 4) - - self.assertEqual(artifacts[0].source, stratum) - self.assertEqual(artifacts[0].name, 'stratum') - self.assertEqual(artifacts[0].dependencies, - [artifacts[1], artifacts[2], artifacts[3]]) - self.assertEqual(artifacts[0].dependents, []) - - self.assertEqual(artifacts[1].source, chunk1) - self.assertEqual(artifacts[1].name, 'chunk1') - self.assertEqual(artifacts[1].dependencies, []) - self.assertEqual(artifacts[1].dependents, - [artifacts[0], artifacts[3]]) - - self.assertEqual(artifacts[2].source, chunk2) - self.assertEqual(artifacts[2].name, 'chunk2') - self.assertEqual(artifacts[2].dependencies, []) - self.assertEqual(artifacts[2].dependents, [artifacts[0], artifacts[3]]) - - self.assertEqual(artifacts[3].source, chunk3) - self.assertEqual(artifacts[3].name, 'chunk3') - self.assertEqual(artifacts[3].dependencies, - [artifacts[1], artifacts[2]]) - self.assertEqual(artifacts[3].dependents, [artifacts[0]]) - - def test_detection_of_invalid_chunk_artifact_references(self): - pool = morphlib.sourcepool.SourcePool() - - morph = FakeChunkMorphology('chunk') - chunk = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') - pool.add(chunk) - - morph = FakeStratumMorphology( - 'stratum', - chunks=[('chunk-runtime', 'chunk', 'repo', 'ref')]) - stratum = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'stratum.morph') - pool.add(stratum) - - self.assertRaises( - morphlib.artifactresolver.UndefinedChunkArtifactError, - self.resolver.resolve_artifacts, pool) + self.assertEqual(len(artifacts), + sum(len(s.split_rules.artifacts) for s in pool)) + + stratum_artifacts = set(a for a in artifacts if a.source == stratum) + chunk_artifacts = [set(a for a in artifacts if a.source == source) + for source in (chunk1, chunk2, chunk3)] + all_chunks = set(itertools.chain.from_iterable(chunk_artifacts)) + + for st_a in stratum_artifacts: + self.assertEqual(st_a.dependents, []) + # This stratum depends on some chunk artifacts + self.assertTrue(any(a in st_a.dependencies for a in all_chunks)) + + for ca in chunk_artifacts[2]: + # There's a stratum dependent on this artifact + self.assertTrue(any(a in stratum_artifacts for a in ca.dependents)) + # chunk3's artifacts depend on chunk1 and chunk2's artifacts + self.assertEqual(set(ca.dependencies), + chunk_artifacts[0] | chunk_artifacts[1]) + + for ca in itertools.chain.from_iterable(chunk_artifacts[0:1]): + self.assertEqual(ca.dependencies, []) + # There's a stratum dependent on this artifact + self.assertTrue(any(a in stratum_artifacts for a in ca.dependents)) + # All chunk3's artifacts depend on this artifact + self.assertTrue(all(c3a in ca.dependents + for c3a in chunk_artifacts[2])) def test_detection_of_mutual_dependency_between_two_strata(self): pool = morphlib.sourcepool.SourcePool() @@ -632,97 +439,6 @@ class ArtifactResolverTests(unittest.TestCase): self.assertRaises(morphlib.artifactresolver.MutualDependencyError, self.resolver.resolve_artifacts, pool) - def test_detection_of_mutual_dependency_between_consecutive_chunks(self): - pool = morphlib.sourcepool.SourcePool() - - morph = FakeStratumMorphology( - 'stratum1', - chunks=[ - ('chunk1', 'chunk1', 'repo', 'original/ref'), - ('chunk2', 'chunk2', 'repo', 'original/ref') - ]) - stratum1 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'stratum1.morph') - pool.add(stratum1) - - morph = FakeStratumMorphology( - 'stratum2', - chunks=[ - ('chunk2', 'chunk2', 'repo', 'original/ref'), - ('chunk1', 'chunk1', 'repo', 'original/ref') - ], - build_depends=[('stratum1', 'repo', 'original/ref')]) - stratum2 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'stratum2.morph') - pool.add(stratum2) - - morph = FakeChunkMorphology('chunk1') - chunk1 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk1.morph') - pool.add(chunk1) - - morph = FakeChunkMorphology('chunk2') - chunk2 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk2.morph') - pool.add(chunk2) - - self.assertRaises(morphlib.artifactresolver.MutualDependencyError, - self.resolver.resolve_artifacts, pool) - - if 0: - # This situation is currently not possible - def test_graceful_handling_of_self_dependencies_of_chunks(self): - pool = morphlib.sourcepool.SourcePool() - - morph = morphlib.morph2.Morphology( - ''' - { - "name": "stratum", - "kind": "stratum", - "chunks": [ - { - "alias": "same-chunk-runtime", - "name": "chunk-runtime", - "morph": "chunk", - "repo": "repo", - "ref": "original/ref" - }, - { - "name": "chunk-runtime", - "morph": "chunk", - "repo": "repo", - "ref": "original/ref", - "build-depends": [ - "same-chunk-runtime" - ] - } - ] - } - ''') - morph.builds_artifacts = ['stratum'] - stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'stratum.morph') - pool.add(stratum) - - morph = FakeChunkMorphology('chunk') - chunk = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk.morph') - pool.add(chunk) - - artifacts = self.resolver.resolve_artifacts(pool) - - self.assertEqual(len(artifacts), 2) - - self.assertEqual(artifacts[0].source, stratum) - self.assertEqual(artifacts[0].name, 'stratum') - self.assertEqual(artifacts[0].dependencies, [artifacts[1]]) - self.assertEqual(artifacts[0].dependents, []) - - self.assertEqual(artifacts[1].source, chunk) - self.assertEqual(artifacts[1].name, 'chunk') - self.assertEqual(artifacts[1].dependencies, []) - self.assertEqual(artifacts[1].dependents, [artifacts[0]]) - def test_detection_of_chunk_dependencies_in_invalid_order(self): pool = morphlib.sourcepool.SourcePool() @@ -796,3 +512,8 @@ class ArtifactResolverTests(unittest.TestCase): self.assertRaises(morphlib.artifactresolver.DependencyFormatError, self.resolver.resolve_artifacts, pool) + + +# TODO: Expand test suite to include better dependency checking, many +# tests were removed due to the fundamental change in how artifacts +# and dependencies are constructed diff --git a/morphlib/artifactsplitrule.py b/morphlib/artifactsplitrule.py new file mode 100644 index 00000000..246691d8 --- /dev/null +++ b/morphlib/artifactsplitrule.py @@ -0,0 +1,303 @@ +# Copyright (C) 2013-2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import collections +import itertools +import re + +import morphlib + + +class Rule(object): + '''Rule base class. + + Rules are passed an object and are expected to determine whether + it matches. It's roughly the same machinery for matching files + as artifacts, it's just that Files are given just the path, while + Artifact matches are given the artifact name and the name of the + source it came from. + + ''' + + def match(self, *args): + return True + + +class FileMatch(Rule): + '''Match a file path against a list of regular expressions. + + If the path matches any of the regular expressions, then the file + is counted as a valid match. + + ''' + + def __init__(self, regexes): + # Possible optimisation: compile regexes as one pattern + self._regexes = [re.compile(r) for r in regexes] + + def match(self, path): + return any(r.match(path) for r in self._regexes) + + +class ArtifactMatch(Rule): + '''Match an artifact's name against a list of regular expressions. + ''' + + def __init__(self, regexes): + # Possible optimisation: compile regexes as one pattern + self._regexes = [re.compile(r) for r in regexes] + + def match(self, (source_name, artifact_name)): + return any(r.match(artifact_name) for r in self._regexes) + + +class ArtifactAssign(Rule): + '''Match only artifacts with the specified source and artifact names. + + This is a valid match if the source and artifact names exactly match. + This is used for explicit artifact assignment e.g. chunk artifact + foo-bins which comes from chunk source foo goes into stratum + bar-runtime. + + ''' + + def __init__(self, source_name, artifact_name): + self._key = (source_name, artifact_name) + + def match(self, (source_name, artifact_name)): + return (source_name, artifact_name) == self._key + + +class SourceAssign(Rule): + '''Match only artifacts which come from the specified source. + + This is a valid match only if the artifact comes from the specified + source. e.g. all artifacts produced by source bar-runtime go into + system baz + + ''' + + def __init__(self, source_name): + self._source = source_name + + def match(self, (source_name, artifact_name)): + return source_name == self._source + + +class SplitRules(collections.Iterable): + '''Rules engine for splitting a source's artifacts. + + Rules are added with the .add(artifact, rule) method, though another + SplitRules may be created by passing a SplitRules to the constructor. + + .match(path|(source, artifact)) and .partition(iterable) are used + to determine if an artifact matches the rules. Rules are processed + in order, so more specific matches first can be followed by more + generic catch-all matches. + + ''' + + def __init__(self, *args): + self._rules = list(*args) + + def __iter__(self): + return iter(self._rules) + + def add(self, artifact, rule): + self._rules.append((artifact, rule)) + + @property + def artifacts(self): + '''Get names of all artifacts in the rule set. + + Returns artifact names in the order they were added to the rules, + and not repeating the artifact. + + ''' + + seen = set() + result = [] + for artifact_name, rule in self._rules: + if artifact_name not in seen: + seen.add(artifact_name) + result.append(artifact_name) + return result + + def match(self, *args): + '''Return all artifact names the given argument matches. + + It's returned in match order as a list, so it's possible to + detect overlapping matches, even though most of the time, the + only used entry will be the first. + + ''' + + return [a for a, r in self._rules if r.match(*args)] + + def partition(self, iterable): + '''Match many files or artifacts. + + It's the common case to take a bunch of filenames and determine + which artifact each should go to, so rather than implement this + logic in multiple places, it's here as a convenience method. + + ''' + + matches = collections.defaultdict(list) + overlaps = collections.defaultdict(set) + unmatched = set() + + for arg in iterable: + matched = self.match(arg) + if len(matched) == 0: + unmatched.add(arg) + continue + if len(matched) != 1: + overlaps[arg].update(matched) + matches[matched[0]].append(arg) + + return matches, overlaps, unmatched + + +# TODO: Work out a good way to feed new defaults in. This is good for +# the usual Linux userspace, but we may find issues and need a +# migration path to a more useful set, or develop a system with +# a different layout, like Android. +DEFAULT_CHUNK_RULES = [ + ('-bins', [ r"(usr/)?s?bin/.*" ]), + ('-libs', [ + r"(usr/)?lib(32|64)?/lib[^/]*\.so(\.\d+)*", + r"(usr/)libexec/.*"]), + ('-devel', [ + r"(usr/)?include/.*", + r"(usr/)?lib(32|64)?/lib.*\.a", + r"(usr/)?lib(32|64)?/lib.*\.la", + r"(usr/)?(lib(32|64)?|share)/pkgconfig/.*\.pc"]), + ('-doc', [ + r"(usr/)?share/doc/.*", + r"(usr/)?share/man/.*", + r"(usr/)?share/info/.*"]), + ('-locale', [ + r"(usr/)?share/locale/.*", + r"(usr/)?share/i18n/.*", + r"(usr/)?share/zoneinfo/.*"]), + ('-misc', [ r".*" ]), +] + + +DEFAULT_STRATUM_RULES = [ + ('-devel', [ + r'.*-devel', + r'.*-debug', + r'.*-doc']), + ('-runtime', [ + r'.*-bins', + r'.*-libs', + r'.*-locale', + r'.*-misc', + r'.*']), +] + + +def unify_chunk_matches(morphology): + '''Create split rules including defaults and per-chunk rules. + + With rules specified in the morphology's 'products' field and the + default rules for chunks, generate rules to match the files produced + by building the chunk to the chunk artifact they should be put in. + + ''' + + split_rules = SplitRules() + + for ca_name, patterns in ((d['artifact'], d['include']) + for d in morphology['products']): + split_rules.add(ca_name, FileMatch(patterns)) + + name = morphology['name'] + for suffix, patterns in DEFAULT_CHUNK_RULES: + ca_name = name + suffix + # Default rules are replaced by explicit ones + if ca_name in split_rules.artifacts: + break + split_rules.add(ca_name, FileMatch(patterns)) + + return split_rules + + +def unify_stratum_matches(morphology): + '''Create split rules including defaults and per-stratum rules. + + With rules specified in the chunk spec's 'artifacts' fields, the + stratum's 'products' field and the default rules for strata, generate + rules to match the artifacts produced by building the chunks in the + strata to the stratum artifact they should be put in. + + ''' + + assignment_split_rules = SplitRules() + for spec in morphology['chunks']: + source_name = spec['name'] + for ca_name, sta_name in sorted(spec.get('artifacts', {}).iteritems()): + assignment_split_rules.add(sta_name, + ArtifactAssign(source_name, ca_name)) + + # Construct match rules separately, so we can use the SplitRules object's + # own knowledge of which rules already exist to determine whether + # to include the default rule. + # Rather than use the existing SplitRules, use a new one, since + # match rules suppliment assignment rules, rather than replace. + match_split_rules = SplitRules() + for sta_name, patterns in ((d['artifact'], d['include']) + for d in morphology.get('products', {})): + match_split_rules.add(sta_name, ArtifactMatch(patterns)) + + for suffix, patterns in DEFAULT_STRATUM_RULES: + sta_name = morphology['name'] + suffix + if sta_name in match_split_rules.artifacts: + break + match_split_rules.add(sta_name, ArtifactMatch(patterns)) + + # Construct a new SplitRules with the assignments before matches + return SplitRules(itertools.chain(assignment_split_rules, + match_split_rules)) + + +def unify_system_matches(morphology): + '''Create split rules including defaults and per-chunk rules. + + With rules specified in the morphology's 'products' field and the + default rules for chunks, generate rules to match the files produced + by building the chunk to the chunk artifact they should be put in. + + ''' + + name = morphology['name'] + '-rootfs' + split_rules = SplitRules() + + for spec in morphology['strata']: + source_name = spec.get('name', spec['morph']) + if spec.get('artifacts', None) is None: + split_rules.add(name, SourceAssign(source_name)) + continue + for sta_name in spec['artifacts']: + split_rules.add(name, ArtifactAssign(source_name, sta_name)) + + return split_rules + + +def unify_cluster_matches(_): + return None diff --git a/morphlib/bins.py b/morphlib/bins.py index 6fb7dc5a..23e3b812 100644 --- a/morphlib/bins.py +++ b/morphlib/bins.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -50,60 +50,7 @@ def safe_makefile(self, tarinfo, targetpath): tarfile.TarFile.makefile = safe_makefile -def _chunk_filenames(rootdir, regexps, dump_memory_profile=None): - - '''Return the filenames for a chunk from the contents of a directory. - - Only files and directories that match at least one of the regular - expressions are accepted. The regular expressions are implicitly - anchored to the beginning of the string, but not the end. The - filenames are relative to rootdir. - - ''' - - dump_memory_profile = dump_memory_profile or (lambda msg: None) - - def matches(filename): - return any(x.match(filename) for x in compiled) - - def names_to_root(filename): - yield filename - while filename != rootdir: - filename = os.path.dirname(filename) - yield filename - - compiled = [re.compile(x) for x in regexps] - include = set() - for dirname, subdirs, basenames in os.walk(rootdir): - subdirpaths = [os.path.join(dirname, x) for x in subdirs] - subdirsymlinks = [x for x in subdirpaths if os.path.islink(x)] - filenames = [os.path.join(dirname, x) for x in basenames] - for filename in [dirname] + subdirsymlinks + filenames: - if matches(os.path.relpath(filename, rootdir)): - for name in names_to_root(filename): - if name not in include: - include.add(name) - else: - logging.debug('regexp MISMATCH: %s' % filename) - dump_memory_profile('after walking') - - return sorted(include) # get dirs before contents - - -def chunk_contents(rootdir, regexps): - ''' Return the list of files in a chunk, with the rootdir - stripped off. - - ''' - - filenames = _chunk_filenames(rootdir, regexps) - # The first entry is the rootdir directory, which we don't need - filenames.pop(0) - contents = [str[len(rootdir):] for str in filenames] - return contents - - -def create_chunk(rootdir, f, regexps, dump_memory_profile=None): +def create_chunk(rootdir, f, include, dump_memory_profile=None): '''Create a chunk from the contents of a directory. ``f`` is an open file handle, to which the tar file is written. @@ -118,14 +65,15 @@ def create_chunk(rootdir, f, regexps, dump_memory_profile=None): # does not complain about an implausibly old timestamp. normalized_timestamp = 683074800 - include = _chunk_filenames(rootdir, regexps, dump_memory_profile) dump_memory_profile('at beginning of create_chunk') + path_pairs = [(relname, os.path.join(rootdir, relname)) + for relname in include] tar = tarfile.open(fileobj=f, mode='w') - for filename in include: + for relname, filename in path_pairs: # Normalize mtime for everything. tarinfo = tar.gettarinfo(filename, - arcname=os.path.relpath(filename, rootdir)) + arcname=relname) tarinfo.ctime = normalized_timestamp tarinfo.mtime = normalized_timestamp if tarinfo.isreg(): @@ -135,11 +83,9 @@ def create_chunk(rootdir, f, regexps, dump_memory_profile=None): tar.addfile(tarinfo) tar.close() - include.remove(rootdir) - for filename in reversed(include): + for relname, filename in reversed(path_pairs): if os.path.isdir(filename) and not os.path.islink(filename): - if not os.listdir(filename): - os.rmdir(filename) + continue else: os.remove(filename) dump_memory_profile('after removing in create_chunks') diff --git a/morphlib/bins_tests.py b/morphlib/bins_tests.py index a9a94a44..60361ece 100644 --- a/morphlib/bins_tests.py +++ b/morphlib/bins_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -107,42 +107,33 @@ class ChunkTests(BinsTest): self.instdir_orig_files = self.recursive_lstat(self.instdir) - def create_chunk(self, regexps): + def create_chunk(self, includes): self.populate_instdir() - morphlib.bins.create_chunk(self.instdir, self.chunk_f, regexps) + morphlib.bins.create_chunk(self.instdir, self.chunk_f, includes) self.chunk_f.flush() - def chunk_contents(self, regexps): - self.populate_instdir() - return morphlib.bins.chunk_contents(self.instdir, regexps) - def unpack_chunk(self): os.mkdir(self.unpacked) morphlib.bins.unpack_binary(self.chunk_file, self.unpacked) - def test_empties_everything(self): - self.create_chunk(['.']) + def test_empties_files(self): + self.create_chunk(['bin/foo', 'lib/libfoo.so']) self.assertEqual([x for x, y in self.recursive_lstat(self.instdir)], - ['.']) + ['.', 'bin', 'lib']) def test_creates_and_unpacks_chunk_exactly(self): - self.create_chunk(['.']) + self.create_chunk(['bin', 'bin/foo', 'lib', 'lib/libfoo.so']) self.unpack_chunk() self.assertEqual(self.instdir_orig_files, self.recursive_lstat(self.unpacked)) def test_uses_only_matching_names(self): - self.create_chunk(['bin']) + self.create_chunk(['bin/foo']) self.unpack_chunk() self.assertEqual([x for x, y in self.recursive_lstat(self.unpacked)], ['.', 'bin', 'bin/foo']) self.assertEqual([x for x, y in self.recursive_lstat(self.instdir)], - ['.', 'lib', 'lib/libfoo.so']) - - def test_list_chunk_contents(self): - contents = self.chunk_contents(['.']) - self.assertEqual(contents, - ['/bin', '/bin/foo', '/lib', '/lib/libfoo.so']) + ['.', 'bin', 'lib', 'lib/libfoo.so']) def test_does_not_compress_artifact(self): self.create_chunk(['bin']) @@ -176,13 +167,13 @@ class ExtractTests(unittest.TestCase): with open(os.path.join(basedir, 'babar'), 'w') as f: pass os.symlink('babar', os.path.join(basedir, 'bar')) - return ['.'] + return ['babar'] linktar = self.create_chunk(make_linkfile) def make_file(basedir): with open(os.path.join(basedir, 'bar'), 'w') as f: pass - return ['.'] + return ['bar'] filetar = self.create_chunk(make_file) os.mkdir(self.unpacked) @@ -194,12 +185,12 @@ class ExtractTests(unittest.TestCase): def test_extracted_dirs_keep_links(self): def make_usrlink(basedir): os.symlink('.', os.path.join(basedir, 'usr')) - return ['.'] + return ['usr'] linktar = self.create_chunk(make_usrlink) def make_usrdir(basedir): os.mkdir(os.path.join(basedir, 'usr')) - return ['.'] + return ['usr'] dirtar = self.create_chunk(make_usrdir) morphlib.bins.unpack_binary_from_file(linktar, self.unpacked) @@ -210,14 +201,14 @@ class ExtractTests(unittest.TestCase): def test_extracted_files_follow_links(self): def make_usrlink(basedir): os.symlink('.', os.path.join(basedir, 'usr')) - return ['.'] + return ['usr'] linktar = self.create_chunk(make_usrlink) def make_usrdir(basedir): os.mkdir(os.path.join(basedir, 'usr')) with open(os.path.join(basedir, 'usr', 'foo'), 'w') as f: pass - return ['.'] + return ['usr', 'usr/foo'] dirtar = self.create_chunk(make_usrdir) morphlib.bins.unpack_binary_from_file(linktar, self.unpacked) diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 4b3b2108..6485f510 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -22,6 +22,14 @@ import tempfile import morphlib +class MultipleRootArtifactsError(morphlib.Error): + + def __init__(self, artifacts): + self.msg = ('System build has multiple root artifacts: %r' + % [a.name for a in artifacts]) + self.artifacts = artifacts + + class BuildCommand(object): '''High level logic for building. @@ -142,7 +150,15 @@ class BuildCommand(object): artifacts = ar.resolve_artifacts(srcpool) self.app.status(msg='Computing build order', chatty=True) - root_artifact = self._find_root_artifact(artifacts) + root_artifacts = self._find_root_artifacts(artifacts) + if len(root_artifacts) > 1: + # Validate root artifacts, since validation covers errors + # such as trying to build a chunk or stratum directly, + # and this is one cause for having multiple root artifacts + for root_artifact in root_artifacts: + self._validate_root_artifact(root_artifact) + raise MultipleRootArtifactsError(root_artifacts) + root_artifact = root_artifacts[0] # Validate the root artifact here, since it's a costly function # to finalise it, so any pre finalisation validation is better @@ -231,8 +247,8 @@ class BuildCommand(object): other.morphology['kind'], wanted)) - def _find_root_artifact(self, artifacts): - '''Find the root artifact among a set of artifacts in a DAG. + def _find_root_artifacts(self, artifacts): + '''Find all the root artifacts among a set of artifacts in a DAG. It would be nice if the ArtifactResolver would return its results in a more useful order to save us from needing to do this -- the root object @@ -240,13 +256,7 @@ class BuildCommand(object): ''' - maybe = set(artifacts) - for a in artifacts: - for dep in a.dependencies: - if dep in maybe: - maybe.remove(dep) - assert len(maybe) == 1 - return maybe.pop() + return [a for a in artifacts if not a.dependents] def build_in_order(self, root_artifact): '''Build everything specified in a build order.''' diff --git a/morphlib/builder2.py b/morphlib/builder2.py index bab89aa2..2dca738c 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -422,33 +422,53 @@ class ChunkBuilder(BuilderBase): def assemble_chunk_artifacts(self, destdir): # pragma: no cover built_artifacts = [] filenames = [] + source = self.artifact.source + split_rules = source.split_rules + + def filepaths(destdir): + for dirname, subdirs, basenames in os.walk(destdir): + subdirsymlinks = [os.path.join(dirname, x) for x in subdirs + if os.path.islink(x)] + filenames = [os.path.join(dirname, x) for x in basenames] + for relpath in (os.path.relpath(x, destdir) for x in + [dirname] + subdirsymlinks + filenames): + yield relpath + + with self.build_watch('determine-splits'): + matches, overlaps, unmatched = \ + split_rules.partition(filepaths(destdir)) + with self.build_watch('create-chunks'): - specs = self.artifact.source.morphology['chunks'] - if len(specs) == 0: - specs = { - self.artifact.source.morphology['name']: ['.'], - } - names = specs.keys() - names.sort(key=lambda name: [ord(c) for c in name]) - for artifact_name in names: - artifact = self.new_artifact(artifact_name) - patterns = specs[artifact_name] - patterns += [r'baserock/%s\.' % artifact_name] - - with self.local_artifact_cache.put(artifact) as f: - contents = morphlib.bins.chunk_contents(destdir, patterns) - self.write_metadata(destdir, artifact_name, contents) - - self.app.status(msg='assembling chunk %s' % artifact_name, - chatty=True) - self.app.status(msg='assembling into %s' % f.name, - chatty=True) + for chunk_artifact_name, chunk_artifact \ + in source.artifacts.iteritems(): + file_paths = matches[chunk_artifact_name] + chunk_artifact = source.artifacts[chunk_artifact_name] + + def all_parents(path): + while path != '': + yield path + path = os.path.dirname(path) + + def parentify(filenames): + names = set() + for name in filenames: + names.update(all_parents(name)) + return sorted(names) + + parented_paths = \ + parentify(file_paths + + ['baserock/%s.meta' % chunk_artifact_name]) + + with self.local_artifact_cache.put(chunk_artifact) as f: + self.write_metadata(destdir, chunk_artifact_name, + parented_paths) + self.app.status(msg='Creating chunk artifact %(name)s', - name=artifact.name) - morphlib.bins.create_chunk(destdir, f, patterns) - built_artifacts.append(artifact) + name=chunk_artifact_name) + morphlib.bins.create_chunk(destdir, f, parented_paths) + built_artifacts.append(chunk_artifact) - files = os.listdir(destdir) + for dirname, subdirs, files in os.walk(destdir): if files: raise Exception('DESTDIR %s is not empty: %s' % (destdir, files)) @@ -458,8 +478,8 @@ class ChunkBuilder(BuilderBase): s = self.artifact.source extract_sources(self.app, self.repo_cache, s.repo, s.sha1, srcdir) -class StratumBuilder(BuilderBase): +class StratumBuilder(BuilderBase): '''Build stratum artifacts.''' def is_constituent(self, artifact): # pragma: no cover @@ -495,16 +515,14 @@ class StratumBuilder(BuilderBase): with self.build_watch('create-chunk-list'): lac = self.local_artifact_cache - artifact_name = self.artifact.source.morphology['name'] - artifact = self.new_artifact(artifact_name) - contents = [x.name for x in constituents] - meta = self.create_metadata(artifact_name, contents) - with lac.put_artifact_metadata(artifact, 'meta') as f: + meta = self.create_metadata(self.artifact.name, + [x.name for x in constituents]) + with lac.put_artifact_metadata(self.artifact, 'meta') as f: json.dump(meta, f, indent=4, sort_keys=True) - with self.local_artifact_cache.put(artifact) as f: + with self.local_artifact_cache.put(self.artifact) as f: json.dump([c.basename() for c in constituents], f) self.save_build_times() - return [artifact] + return [self.artifact] class SystemBuilder(BuilderBase): # pragma: no cover diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index d7e2e3b1..2312abc3 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -90,6 +90,8 @@ class CacheKeyComputer(object): keys['build-mode'] = artifact.source.build_mode keys['prefix'] = artifact.source.prefix keys['tree'] = artifact.source.tree + keys['split-rules'] = [(a, [rgx.pattern for rgx in r._regexes]) + for (a, r) in artifact.source.split_rules] elif kind in ('system', 'stratum'): morphology = artifact.source.morphology le_dict = dict((k, morphology[k]) for k in morphology.keys()) diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py index 2f033a7a..4e73e905 100644 --- a/morphlib/cachekeycomputer_tests.py +++ b/morphlib/cachekeycomputer_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -53,7 +53,8 @@ class CacheKeyComputerTests(unittest.TestCase): { "name": "chunk", "repo": "repo", - "ref": "original/ref" + "ref": "original/ref", + "build-depends": [] } ] }''', @@ -64,12 +65,14 @@ class CacheKeyComputerTests(unittest.TestCase): { "name": "chunk2", "repo": "repo", - "ref": "original/ref" + "ref": "original/ref", + "build-depends": [] }, { "name": "chunk3", "repo": "repo", - "ref": "original/ref" + "ref": "original/ref", + "build-depends": [] } ] }''', @@ -118,7 +121,6 @@ class CacheKeyComputerTests(unittest.TestCase): for artifact in self.artifacts: if artifact.name == name: return artifact - raise def test_compute_key_hashes_all_types(self): runcount = {'thing': 0, 'dict': 0, 'list': 0, 'tuple': 0} @@ -184,8 +186,8 @@ class CacheKeyComputerTests(unittest.TestCase): self.assertEqual(old_sha, new_sha) def test_same_morphology_added_to_source_pool_only_appears_once(self): - src = morphlib.source.Source('repo', 'original/ref', 'sha', 'tree', - '{"name": "chunk", "kind": "chunk"}', + m = morphlib.morph2.Morphology('{"name": "chunk", "kind": "chunk"}') + src = morphlib.source.Source('repo', 'original/ref', 'sha', 'tree', m, 'chunk.morph') sp = morphlib.sourcepool.SourcePool() sp.add(src) diff --git a/morphlib/localartifactcache_tests.py b/morphlib/localartifactcache_tests.py index d7743359..18d20612 100644 --- a/morphlib/localartifactcache_tests.py +++ b/morphlib/localartifactcache_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012,2013 Codethink Limited +# Copyright (C) 2012,2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -30,7 +30,7 @@ class LocalArtifactCacheTests(unittest.TestCase): morph = morphlib.morph2.Morphology( ''' { - "chunk": "chunk", + "name": "chunk", "kind": "chunk", "artifacts": { "chunk-runtime": [ diff --git a/morphlib/morph2.py b/morphlib/morph2.py index 0e0d9201..fd72aa94 100644 --- a/morphlib/morph2.py +++ b/morphlib/morph2.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,7 +14,6 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -import copy import re import morphlib @@ -45,7 +44,7 @@ class Morphology(object): ('install-commands', None), ('post-install-commands', None), ('devices', None), - ('chunks', []), + ('products', []), ('max-jobs', None), ('build-system', 'manual') ], @@ -87,6 +86,14 @@ class Morphology(object): def __contains__(self, key): return key in self._dict + # Not covered by tests, since it's trivial, morph2 is going away + # and only exists so the new morphology validation code can use it. + def get(self, key, default=None): # pragma: no cover + try: + return self[key] + except KeyError: + return default + def get_commands(self, which): '''Return the commands to run from a morphology or the build system''' if self[which] is None: diff --git a/morphlib/morph2_tests.py b/morphlib/morph2_tests.py index aaa1d1cc..ba90313f 100644 --- a/morphlib/morph2_tests.py +++ b/morphlib/morph2_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -50,7 +50,7 @@ class MorphologyTests(unittest.TestCase): self.assertEqual(m['install-commands'], None) self.assertEqual(m['post-install-commands'], None) self.assertEqual(m['max-jobs'], None) - self.assertEqual(m['chunks'], []) + self.assertEqual(m['products'], []) if morphlib.got_yaml: def test_parses_simple_yaml_chunk(self): @@ -76,7 +76,7 @@ class MorphologyTests(unittest.TestCase): self.assertEqual(m['install-commands'], None) self.assertEqual(m['post-install-commands'], None) self.assertEqual(m['max-jobs'], None) - self.assertEqual(m['chunks'], []) + self.assertEqual(m['products'], []) def test_sets_stratum_chunks_repo_and_morph_from_name(self): m = Morphology(''' diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py index e7c1d9ff..637544be 100644 --- a/morphlib/morphloader.py +++ b/morphlib/morphloader.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -44,16 +44,33 @@ class UnknownKindError(morphlib.Error): class MissingFieldError(morphlib.Error): - def __init__(self, field, morphology): + def __init__(self, field, morphology_name): + self.field = field + self.morphology_name = morphology_name self.msg = ( - 'Missing field %s from morphology %s' % (field, morphology)) + 'Missing field %s from morphology %s' % (field, morphology_name)) class InvalidFieldError(morphlib.Error): - def __init__(self, field, morphology): + def __init__(self, field, morphology_name): + self.field = field + self.morphology_name = morphology_name self.msg = ( - 'Field %s not allowed in morphology %s' % (field, morphology)) + 'Field %s not allowed in morphology %s' % (field, morphology_name)) + + +class InvalidTypeError(morphlib.Error): + + def __init__(self, field, expected, actual, morphology_name): + self.field = field + self.expected = expected + self.actual = actual + self.morphology_name = morphology_name + self.msg = ( + 'Field %s expected type %s, got %s in morphology %s' % + (field, expected, actual, morphology_name)) + class ObsoleteFieldsError(morphlib.Error): @@ -140,6 +157,16 @@ class EmptySystemError(morphlib.Error): self, 'System %(system_name)s has no strata.' % locals()) +class MultipleValidationErrors(morphlib.Error): + + def __init__(self, name, errors): + self.name = name + self.errors = errors + self.msg = 'Multiple errors when validating %(name)s:' + for error in errors: + self.msg += ('\t' + str(error)) + + class MorphologyLoader(object): '''Load morphologies from disk, or save them back to disk.''' @@ -185,7 +212,7 @@ class MorphologyLoader(object): 'install-commands': [], 'post-install-commands': [], 'devices': [], - 'chunks': [], + 'products': [], 'max-jobs': None, 'build-system': 'manual', }, @@ -193,6 +220,7 @@ class MorphologyLoader(object): 'chunks': [], 'description': '', 'build-depends': [], + 'products': [], }, 'system': { 'description': '', @@ -356,8 +384,84 @@ class MorphologyLoader(object): spec.get('alias', spec['name']), morph.filename) - def _validate_chunk(self, morph): - pass + @classmethod + def _validate_chunk(cls, morphology): + errors = [] + + if 'products' in morphology: + cls._validate_products(morphology['name'], + morphology['products'], errors) + + if len(errors) == 1: + raise errors[0] + elif errors: + raise MultipleValidationErrors(morphology['name'], errors) + + @classmethod + def _validate_products(cls, morphology_name, products, errors): + '''Validate the products field is of the correct type.''' + if (not isinstance(products, collections.Iterable) + or isinstance(products, collections.Mapping)): + raise InvalidTypeError('products', list, + type(products), morphology_name) + + for spec_index, spec in enumerate(products): + + if not isinstance(spec, collections.Mapping): + e = InvalidTypeError('products[%d]' % spec_index, + dict, type(spec), morphology_name) + errors.append(e) + continue + + cls._validate_products_spec_fields_exist(morphology_name, + spec_index, spec, errors) + + if 'include' in spec: + cls._validate_products_specs_include( + morphology_name, spec_index, spec['include'], errors) + + product_spec_required_fields = ('artifact', 'include') + @classmethod + def _validate_products_spec_fields_exist( + cls, morphology_name, spec_index, spec, errors): + + given_fields = sorted(spec.iterkeys()) + missing = (field for field in cls.product_spec_required_fields + if field not in given_fields) + for field in missing: + e = MissingFieldError('products[%d].%s' % (spec_index, field), + morphology_name) + errors.append(e) + unexpected = (field for field in given_fields + if field not in cls.product_spec_required_fields) + for field in unexpected: + e = InvalidFieldError('products[%d].%s' % (spec_index, field), + morphology_name) + errors.append(e) + + @classmethod + def _validate_products_specs_include(cls, morphology_name, spec_index, + include_patterns, errors): + '''Validate that products' include field is a list of strings.''' + # Allow include to be most iterables, but not a mapping + # or a string, since iter of a mapping is just the keys, + # and the iter of a string is a 1 character length string, + # which would also validate as an iterable of strings. + if (not isinstance(include_patterns, collections.Iterable) + or isinstance(include_patterns, collections.Mapping) + or isinstance(include_patterns, basestring)): + + e = InvalidTypeError('products[%d].include' % spec_index, list, + type(include_patterns), morphology_name) + errors.append(e) + else: + for pattern_index, pattern in enumerate(include_patterns): + pattern_path = ('products[%d].include[%d]' % + (spec_index, pattern_index)) + if not isinstance(pattern, basestring): + e = InvalidTypeError(pattern_path, str, + type(pattern), morphology_name) + errors.append(e) def _require_field(self, field, morphology): if field not in morphology: diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py index 8b87467a..c2fbc5e8 100644 --- a/morphlib/morphloader_tests.py +++ b/morphlib/morphloader_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -78,6 +78,99 @@ build-system: dummy self.assertRaises( morphlib.morphloader.InvalidFieldError, self.loader.validate, m) + def test_validate_requires_products_list(self): + m = morphlib.morph3.Morphology( + kind='chunk', + name='foo', + products={ + 'foo-runtime': ['.'], + 'foo-devel': ['.'], + }) + with self.assertRaises(morphlib.morphloader.InvalidTypeError) as cm: + self.loader.validate(m) + e = cm.exception + self.assertEqual(e.field, 'products') + self.assertEqual(e.expected, list) + self.assertEqual(e.actual, dict) + self.assertEqual(e.morphology_name, 'foo') + + def test_validate_requires_products_list_of_mappings(self): + m = morphlib.morph3.Morphology( + kind='chunk', + name='foo', + products=[ + 'foo-runtime', + ]) + with self.assertRaises(morphlib.morphloader.InvalidTypeError) as cm: + self.loader.validate(m) + e = cm.exception + self.assertEqual(e.field, 'products[0]') + self.assertEqual(e.expected, dict) + self.assertEqual(e.actual, str) + self.assertEqual(e.morphology_name, 'foo') + + def test_validate_requires_products_list_required_fields(self): + m = morphlib.morph3.Morphology( + kind='chunk', + name='foo', + products=[ + { + 'factiart': 'foo-runtime', + 'cludein': [], + } + ]) + with self.assertRaises(morphlib.morphloader.MultipleValidationErrors) \ + as cm: + self.loader.validate(m) + exs = cm.exception.errors + self.assertEqual(type(exs[0]), morphlib.morphloader.MissingFieldError) + self.assertEqual(exs[0].field, 'products[0].artifact') + self.assertEqual(type(exs[1]), morphlib.morphloader.MissingFieldError) + self.assertEqual(exs[1].field, 'products[0].include') + self.assertEqual(type(exs[2]), morphlib.morphloader.InvalidFieldError) + self.assertEqual(exs[2].field, 'products[0].cludein') + self.assertEqual(type(exs[3]), morphlib.morphloader.InvalidFieldError) + self.assertEqual(exs[3].field, 'products[0].factiart') + + def test_validate_requires_products_list_include_is_list(self): + m = morphlib.morph3.Morphology( + kind='chunk', + name='foo', + products=[ + { + 'artifact': 'foo-runtime', + 'include': '.*', + } + ]) + with self.assertRaises(morphlib.morphloader.InvalidTypeError) as cm: + self.loader.validate(m) + ex = cm.exception + self.assertEqual(ex.field, 'products[0].include') + self.assertEqual(ex.expected, list) + self.assertEqual(ex.actual, str) + self.assertEqual(ex.morphology_name, 'foo') + + def test_validate_requires_products_list_include_is_list_of_strings(self): + m = morphlib.morph3.Morphology( + kind='chunk', + name='foo', + products=[ + { + 'artifact': 'foo-runtime', + 'include': [ + 123, + ] + } + ]) + with self.assertRaises(morphlib.morphloader.InvalidTypeError) as cm: + self.loader.validate(m) + ex = cm.exception + self.assertEqual(ex.field, 'products[0].include[0]') + self.assertEqual(ex.expected, str) + self.assertEqual(ex.actual, int) + self.assertEqual(ex.morphology_name, 'foo') + + def test_fails_to_validate_stratum_with_no_fields(self): m = morphlib.morph3.Morphology({ 'kind': 'stratum', @@ -438,7 +531,7 @@ name: foo 'pre-install-commands': [], 'post-install-commands': [], - 'chunks': [], + 'products': [], 'devices': [], 'max-jobs': None, }) @@ -491,6 +584,7 @@ name: foo 'build-depends': [], }, ], + 'products': [], }) def test_unsets_defaults_for_strata(self): diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py index 5afafefb..3462dd36 100644 --- a/morphlib/morphologyfactory.py +++ b/morphlib/morphologyfactory.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -167,9 +167,12 @@ class MorphologyFactory(object): def _check_and_tweak_chunk(self, morphology, reponame, sha1, filename): '''Check and tweak a chunk morphology.''' - if 'chunks' in morphology and len(morphology['chunks']) > 1: - morphology.builds_artifacts = morphology['chunks'].keys() + if 'products' in morphology and len(morphology['products']) > 1: + morphology.builds_artifacts = [d['artifact'] + for d in morphology['products']] else: morphology.builds_artifacts = [morphology['name']] morphology.needs_artifact_metadata_cached = False + + morphlib.morphloader.MorphologyLoader._validate_chunk(morphology) diff --git a/morphlib/morphologyfactory_tests.py b/morphlib/morphologyfactory_tests.py index 6e1e67d3..30504e00 100644 --- a/morphlib/morphologyfactory_tests.py +++ b/morphlib/morphologyfactory_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -50,10 +50,16 @@ class FakeLocalRepo(object): "name": "chunk-split", "kind": "chunk", "build-system": "bar", - "chunks": { - "chunk-split-runtime": [], - "chunk-split-devel": [] - } + "products": [ + { + "artifact": "chunk-split-runtime", + "include": [] + }, + { + "artifact": "chunk-split-devel", + "include": [] + } + ] }''', 'stratum.morph': '''{ "name": "stratum", diff --git a/morphlib/remoteartifactcache_tests.py b/morphlib/remoteartifactcache_tests.py index e7f45f58..d11bf264 100644 --- a/morphlib/remoteartifactcache_tests.py +++ b/morphlib/remoteartifactcache_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -27,7 +27,7 @@ class RemoteArtifactCacheTests(unittest.TestCase): morph = morphlib.morph2.Morphology( ''' { - "chunk": "chunk", + "name": "chunk", "kind": "chunk", "artifacts": { "chunk-runtime": [ diff --git a/morphlib/source.py b/morphlib/source.py index 99b0a993..75a2e4de 100644 --- a/morphlib/source.py +++ b/morphlib/source.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -30,6 +30,8 @@ class Source(object): * ``tree`` -- the SHA1 of the tree corresponding to the commit * ``morphology`` -- the in-memory representation of the morphology we use * ``filename`` -- basename of the morphology filename + * ``artifacts`` -- the set of artifacts this source produces. + * ``split_rules`` -- rules for splitting the source's produced artifacts ''' @@ -43,6 +45,13 @@ class Source(object): self.morphology = morphology self.filename = filename + kind = morphology['kind'] + unifier = getattr(morphlib.artifactsplitrule, + 'unify_%s_matches' % kind) + self.split_rules = unifier(morphology) + self.artifacts = {name: morphlib.artifact.Artifact(self, name) + for name in self.split_rules.artifacts} + def __str__(self): # pragma: no cover return '%s|%s|%s' % (self.repo_name, self.original_ref, |