diff options
author | Richard Maw <richard.maw@codethink.co.uk> | 2014-09-22 09:22:56 +0000 |
---|---|---|
committer | Richard Maw <richard.maw@codethink.co.uk> | 2014-09-22 09:22:56 +0000 |
commit | cb7eed0a589c7b2ea62ca6df789422d09b9dcb43 (patch) | |
tree | e6bb5d5c4e06f14de98569298cc00e3fcbbf829f /morphlib | |
parent | e62eced3044c2383de3029e9d7ae2b2649704e80 (diff) | |
parent | 945c60a1aa0b48f49c08e70206a5ca24f1c710bb (diff) | |
download | morph-cb7eed0a589c7b2ea62ca6df789422d09b9dcb43.tar.gz |
Merge branch 'baserock/richardmaw-os/tidy-build-logic-v7'
Reviewed-by: Lars Wirzenius (+2 to misc fixups)
Reviewed-by: Sam Thursfield (+1 to per-source building)
Reviewed-by: Paul Sherwood (+1 to per-source building)
Diffstat (limited to 'morphlib')
27 files changed, 686 insertions, 938 deletions
diff --git a/morphlib/app.py b/morphlib/app.py index 48de6aba..9ab102b3 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -294,9 +294,11 @@ class Morph(cliapp.Application): pool = morphlib.sourcepool.SourcePool() def add_to_pool(reponame, ref, filename, absref, tree, morphology): - source = morphlib.source.Source(reponame, ref, absref, tree, - morphology, filename) - pool.add(source) + sources = morphlib.source.make_sources(reponame, ref, + filename, absref, + tree, morphology) + for source in sources: + pool.add(source) self.traverse_morphs(repo, ref, [filename], lrc, rrc, update=not self.settings['no-git-update'], diff --git a/morphlib/artifact.py b/morphlib/artifact.py index da6d3763..8b4ce65e 100644 --- a/morphlib/artifact.py +++ b/morphlib/artifact.py @@ -22,13 +22,7 @@ class Artifact(object): * ``source`` -- the source from which the artifact is built * ``name`` -- the name of the artifact - * ``cache_key`` -- a cache key to uniquely identify the artifact - * ``cache_id`` -- a dict describing the components of the cache key - * ``dependencies`` -- list of Artifacts that need to be built beforehand - * ``dependents`` -- list of Artifacts that need this Artifact to be built - * ``metadata_version`` -- When the format of the artifact metadata - changes, this version number is raised causing - any existing cached artifacts to be invalidated + * ``dependents`` -- list of Sources that need this Artifact to be built The ``dependencies`` and ``dependents`` lists MUST be modified by the ``add_dependencies`` and ``add_dependent`` methods only. @@ -38,50 +32,21 @@ class Artifact(object): def __init__(self, source, name): self.source = source self.name = name - self.cache_id = None - self.cache_key = None - self.dependencies = [] self.dependents = [] - self.metadata_version = 1 - - def add_dependency(self, artifact): - '''Add ``artifact`` to the dependency list.''' - if artifact not in self.dependencies: - self.dependencies.append(artifact) - artifact.dependents.append(self) - - def depends_on(self, artifact): - '''Do we depend on ``artifact``?''' - return artifact in self.dependencies def basename(self): # pragma: no cover - return '%s.%s.%s' % (self.cache_key, - str(self.source.morphology['kind']), - str(self.name)) + return '%s.%s' % (self.source.basename(), str(self.name)) def metadata_basename(self, metadata_name): # pragma: no cover - return '%s.%s.%s.%s' % (self.cache_key, - str(self.source.morphology['kind']), - str(self.name), - metadata_name) - - def get_dependency_prefix_set(self): - '''Collects all install prefixes of this artifact's build dependencies - - If any of the build dependencies of a chunk artifact are installed - to non-standard prefixes, we need to add those prefixes to the - PATH of the current artifact. - - ''' - result = set() - for d in self.dependencies: - if d.source.morphology['kind'] == 'chunk': - result.add(d.source.prefix) - return result + return '%s.%s' % (self.basename(), metadata_name) def __str__(self): # pragma: no cover return '%s|%s' % (self.source, self.name) + def __repr__(self): # pragma: no cover + return 'Artifact(%s)' % str(self) + + def walk(self): # pragma: no cover '''Return list of an artifact and its build dependencies. @@ -95,7 +60,7 @@ class Artifact(object): def depth_first(a): if a not in done: done.add(a) - for dep in a.dependencies: + for dep in a.source.dependencies: for ret in depth_first(dep): yield ret yield a diff --git a/morphlib/artifact_tests.py b/morphlib/artifact_tests.py index 62b1bfb9..abd8767e 100644 --- a/morphlib/artifact_tests.py +++ b/morphlib/artifact_tests.py @@ -39,13 +39,16 @@ class ArtifactTests(unittest.TestCase): include: - usr/include ''') - self.source = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') + self.source, = morphlib.source.make_sources('repo', 'ref', + 'chunk.morph', 'sha1', + 'tree', morph) self.artifact_name = 'chunk-runtime' - self.artifact = morphlib.artifact.Artifact( - self.source, self.artifact_name) - self.other = morphlib.artifact.Artifact( - self.source, self.artifact_name) + self.artifact = self.source.artifacts[self.artifact_name] + self.other_source, = morphlib.source.make_sources('repo', 'ref', + 'chunk.morph', + 'sha1', 'tree', + morph) + self.other = self.other_source.artifacts[self.artifact_name] def test_constructor_sets_source(self): self.assertEqual(self.artifact.source, self.source) @@ -53,36 +56,5 @@ class ArtifactTests(unittest.TestCase): def test_constructor_sets_name(self): self.assertEqual(self.artifact.name, self.artifact_name) - def test_constructor_initializes_cache_key_as_none(self): - self.assertEqual(self.artifact.cache_key, None) - - def test_sets_dependencies_to_empty(self): - self.assertEqual(self.artifact.dependencies, []) - def test_sets_dependents_to_empty(self): self.assertEqual(self.artifact.dependents, []) - - def test_does_not_depend_on_other_initially(self): - self.assertFalse(self.artifact.depends_on(self.other)) - - def test_adds_dependency(self): - self.artifact.add_dependency(self.other) - self.assertEqual(self.artifact.dependencies, [self.other]) - self.assertEqual(self.other.dependents, [self.artifact]) - self.assertTrue(self.artifact.depends_on(self.other)) - - def test_does_not_add_dependency_twice(self): - self.artifact.add_dependency(self.other) - self.artifact.add_dependency(self.other) - self.assertEqual(self.artifact.dependencies, [self.other]) - self.assertEqual(self.other.dependents, [self.artifact]) - self.assertTrue(self.artifact.depends_on(self.other)) - - def test_get_dependency_prefix(self): - self.artifact.add_dependency(self.other) - self.artifact.source.prefix = '/bar' - self.other.source = copy.copy(self.artifact.source) - self.other.source.prefix = '/foo' - - prefix_set = self.artifact.get_dependency_prefix_set() - self.assertEqual(prefix_set, set(['/foo'])) diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index 9e3cea48..5deb25b7 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -37,15 +37,6 @@ class DependencyOrderError(cliapp.AppException): (stratum_source, chunk, dependency_name)) -class DependencyFormatError(cliapp.AppException): - - def __init__(self, stratum_source, chunk): - cliapp.AppException.__init__( - self, 'In stratum %s, chunk %s uses an invalid ' - 'build-depends format' % (stratum_source, chunk)) - - - class ArtifactResolver(object): '''Resolves sources into artifacts that would be build from the sources. @@ -77,7 +68,7 @@ class ArtifactResolver(object): while queue: source = queue.popleft() - if source.morphology['kind'] == 'system': + if source.morphology['kind'] == 'system': # pragma: no cover systems = [source.artifacts[name] for name in source.split_rules.artifacts] @@ -94,8 +85,11 @@ class ArtifactResolver(object): artifacts.append(artifact) self._added_artifacts.add(artifact) elif source.morphology['kind'] == 'stratum': + # Iterate split_rules.artifacts, rather than + # artifacts.values() to preserve ordering strata = [source.artifacts[name] - for name in source.split_rules.artifacts] + for name in source.split_rules.artifacts + if name in source.artifacts] # If we were not given systems, return the strata here, # rather than have the systems return them. @@ -135,26 +129,30 @@ class ArtifactResolver(object): if x.morphology['kind'] != 'chunk'] return collections.deque(sources) - def _resolve_system_dependencies(self, systems, source, queue): + def _resolve_system_dependencies(self, systems, + source, queue): # pragma: no cover artifacts = [] for info in source.morphology['strata']: - stratum_source = self._source_pool.lookup( + for stratum_source in self._source_pool.lookup( info.get('repo') or source.repo_name, info.get('ref') or source.original_ref, - morphlib.util.sanitise_morphology_path(info['morph'])) - stratum_name = stratum_source.morphology['name'] + morphlib.util.sanitise_morphology_path(info['morph'])): - matches, overlaps, unmatched = source.split_rules.partition( - ((stratum_name, sta_name) for sta_name - in stratum_source.split_rules.artifacts)) - for system in systems: - for (stratum_name, sta_name) in matches[system.name]: - stratum = stratum_source.artifacts[sta_name] - system.add_dependency(stratum) - artifacts.append(stratum) + stratum_morph_name = stratum_source.morphology['name'] + + matches, overlaps, unmatched = source.split_rules.partition( + ((stratum_morph_name, sta_name) for sta_name + in stratum_source.split_rules.artifacts)) + for system in systems: + for (stratum_name, sta_name) in matches[system.name]: + if sta_name in stratum_source.artifacts: + stratum_artifact = \ + stratum_source.artifacts[sta_name] + source.add_dependency(stratum_artifact) + artifacts.append(stratum_artifact) - queue.append(stratum_source) + queue.append(stratum_source) return artifacts @@ -164,28 +162,32 @@ class ArtifactResolver(object): stratum_build_depends = [] for stratum_info in source.morphology.get('build-depends') or []: - other_source = self._source_pool.lookup( + for other_source in self._source_pool.lookup( stratum_info.get('repo') or source.repo_name, stratum_info.get('ref') or source.original_ref, - morphlib.util.sanitise_morphology_path(stratum_info['morph'])) + morphlib.util.sanitise_morphology_path(stratum_info['morph'])): - # Make every stratum artifact this stratum source produces - # depend on every stratum artifact the other stratum source - # produces. - for sta_name in other_source.split_rules.artifacts: - other_stratum = other_source.artifacts[sta_name] + # Make every stratum artifact this stratum source produces + # depend on every stratum artifact the other stratum source + # produces. + for sta_name in other_source.split_rules.artifacts: + # Strata have split rules for artifacts they don't build, + # since they need to know to yield a match to its sibling + if sta_name not in other_source.artifacts: + continue + other_stratum = other_source.artifacts[sta_name] - stratum_build_depends.append(other_stratum) + stratum_build_depends.append(other_stratum) - artifacts.append(other_stratum) + artifacts.append(other_stratum) - for stratum in strata: - if other_stratum.depends_on(stratum): - raise MutualDependencyError(stratum, other_stratum) + for stratum in strata: + if other_source.depends_on(stratum): + raise MutualDependencyError(stratum, other_stratum) - stratum.add_dependency(other_stratum) + source.add_dependency(other_stratum) - queue.append(other_source) + queue.append(other_source) # 'name' here is the chunk artifact name name_to_processed_artifacts = {} @@ -196,9 +198,9 @@ class ArtifactResolver(object): chunk_source = self._source_pool.lookup( info['repo'], info['ref'], - filename) + filename)[0] - chunk_name = chunk_source.morphology['name'] + chunk_name = chunk_source.name # Resolve now to avoid a search for the parent morphology later chunk_source.build_mode = info['build-mode'] @@ -206,39 +208,30 @@ class ArtifactResolver(object): build_depends = info.get('build-depends', None) - for ca_name in chunk_source.split_rules.artifacts: - chunk_artifact = chunk_source.artifacts[ca_name] - - # Add our stratum's build depends as dependencies of this chunk - for other_stratum in stratum_build_depends: - chunk_artifact.add_dependency(other_stratum) + # Add our stratum's build depends as dependencies of this chunk + for other_stratum in stratum_build_depends: + chunk_source.add_dependency(other_stratum) # Add dependencies between chunks mentioned in this stratum - if isinstance(build_depends, list): - for name in build_depends: - if name not in name_to_processed_artifacts: - raise DependencyOrderError( - source, info['name'], name) - other_artifacts = name_to_processed_artifacts[name] - for other_artifact in other_artifacts: - for ca_name in chunk_source.split_rules.artifacts: - chunk_artifact = chunk_source.artifacts[ca_name] - chunk_artifact.add_dependency(other_artifact) - else: - raise DependencyFormatError(source, info['name']) + for name in build_depends: # pragma: no cover + if name not in name_to_processed_artifacts: + raise DependencyOrderError( + source, info['name'], name) + other_artifacts = name_to_processed_artifacts[name] + for other_artifact in other_artifacts: + chunk_source.add_dependency(other_artifact) # Add build dependencies between our stratum's artifacts # and the chunk artifacts produced by this stratum. matches, overlaps, unmatched = source.split_rules.partition( ((chunk_name, ca_name) for ca_name in chunk_source.split_rules.artifacts)) - for stratum in strata: - for (chunk_name, ca_name) in matches[stratum.name]: - chunk_artifact = chunk_source.artifacts[ca_name] - stratum.add_dependency(chunk_artifact) - # Only return chunks required to build strata we need - if chunk_artifact not in artifacts: - artifacts.append(chunk_artifact) + for (chunk_name, ca_name) in matches[source.name]: + chunk_artifact = chunk_source.artifacts[ca_name] + source.add_dependency(chunk_artifact) + # Only return chunks required to build strata we need + if chunk_artifact not in artifacts: + artifacts.append(chunk_artifact) # Add these chunks to the processed artifacts, so other diff --git a/morphlib/artifactresolver_tests.py b/morphlib/artifactresolver_tests.py index 96f7ced8..89f30010 100644 --- a/morphlib/artifactresolver_tests.py +++ b/morphlib/artifactresolver_tests.py @@ -33,15 +33,12 @@ def get_chunk_morphology(name, artifact_names=[]): text = yaml.dump({"name": name, "kind": "chunk", "products": artifacts}, default_flow_style=False) - builds_artifacts = artifact_names else: text = yaml.dump({'name': name, 'kind': 'chunk'}, default_flow_style=False) - builds_artifacts = [name] loader = morphlib.morphloader.MorphologyLoader() morph = loader.load_from_string(text) - morph.builds_artifacts = builds_artifacts return morph def get_stratum_morphology(name, chunks=[], build_depends=[]): @@ -75,7 +72,6 @@ def get_stratum_morphology(name, chunks=[], build_depends=[]): loader = morphlib.morphloader.MorphologyLoader() morph = loader.load_from_string(text) - morph.builds_artifacts = [name] return morph @@ -93,9 +89,11 @@ class ArtifactResolverTests(unittest.TestCase): pool = morphlib.sourcepool.SourcePool() morph = get_chunk_morphology('chunk') - source = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') - pool.add(source) + sources = morphlib.source.make_sources('repo', 'ref', + 'chunk.morph', 'sha1', + 'tree', morph) + for source in sources: + pool.add(source) artifacts = self.resolver.resolve_artifacts(pool) @@ -105,16 +103,18 @@ class ArtifactResolverTests(unittest.TestCase): for artifact in artifacts: self.assertEqual(artifact.source, source) self.assertTrue(artifact.name.startswith('chunk')) - self.assertEqual(artifact.dependencies, []) + self.assertEqual(source.dependencies, []) self.assertEqual(artifact.dependents, []) def test_resolve_single_chunk_with_one_new_artifact(self): pool = morphlib.sourcepool.SourcePool() morph = get_chunk_morphology('chunk', ['chunk-foobar']) - source = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') - pool.add(source) + sources = morphlib.source.make_sources('repo', 'ref', + 'chunk.morph', 'sha1', + 'tree', morph) + for source in sources: + pool.add(source) artifacts = self.resolver.resolve_artifacts(pool) @@ -123,16 +123,18 @@ class ArtifactResolverTests(unittest.TestCase): foobartifact, = (a for a in artifacts if a.name == 'chunk-foobar') self.assertEqual(foobartifact.source, source) - self.assertEqual(foobartifact.dependencies, []) + self.assertEqual(foobartifact.source.dependencies, []) self.assertEqual(foobartifact.dependents, []) def test_resolve_single_chunk_with_two_new_artifacts(self): pool = morphlib.sourcepool.SourcePool() morph = get_chunk_morphology('chunk', ['chunk-baz', 'chunk-qux']) - source = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') - pool.add(source) + sources = morphlib.source.make_sources('repo', 'ref', + 'chunk.morph', 'sha1', + 'tree', morph) + for source in sources: + pool.add(source) artifacts = self.resolver.resolve_artifacts(pool) artifacts.sort(key=lambda a: a.name) @@ -143,254 +145,109 @@ class ArtifactResolverTests(unittest.TestCase): for name in ('chunk-baz', 'chunk-qux'): artifact, = (a for a in artifacts if a.name == name) self.assertEqual(artifact.source, source) - self.assertEqual(artifact.dependencies, []) + self.assertEqual(artifact.source.dependencies, []) self.assertEqual(artifact.dependents, []) def test_resolve_stratum_and_chunk(self): pool = morphlib.sourcepool.SourcePool() morph = get_chunk_morphology('chunk') - chunk = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') - pool.add(chunk) + sources = morphlib.source.make_sources('repo', 'ref', + 'chunk.morph', 'sha1', + 'tree', morph) + for chunk in sources: + pool.add(chunk) morph = get_stratum_morphology( 'stratum', chunks=[('chunk', 'chunk', 'repo', 'ref')]) - stratum = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'stratum.morph') - pool.add(stratum) + stratum_sources = set(morphlib.source.make_sources('repo', 'ref', + 'stratum.morph', + 'sha1', 'tree', + morph)) + for stratum in stratum_sources: + pool.add(stratum) artifacts = self.resolver.resolve_artifacts(pool) + all_artifacts = set() + for s in pool: all_artifacts.update(s.split_rules.artifacts) + + self.assertEqual(set(a.name for a in artifacts), all_artifacts) self.assertEqual(len(artifacts), - sum(len(s.split_rules.artifacts) for s in pool)) + len(all_artifacts)) + - stratum_artifacts = set(a for a in artifacts if a.source == stratum) + stratum_artifacts = set(a for a in artifacts + if a.source in stratum_sources) chunk_artifacts = set(a for a in artifacts if a.source == chunk) for stratum_artifact in stratum_artifacts: self.assertTrue(stratum_artifact.name.startswith('stratum')) self.assertEqual(stratum_artifact.dependents, []) - self.assertTrue(any(dep in chunk_artifacts - for dep in stratum_artifact.dependencies)) + self.assertTrue( + any(dep in chunk_artifacts + for dep in stratum_artifact.source.dependencies)) for chunk_artifact in chunk_artifacts: self.assertTrue(chunk_artifact.name.startswith('chunk')) - self.assertEqual(chunk_artifact.dependencies, []) - self.assertTrue(any(dep in stratum_artifacts + self.assertEqual(chunk_artifact.source.dependencies, []) + self.assertTrue(any(dep in stratum_sources for dep in chunk_artifact.dependents)) def test_resolve_stratum_and_chunk_with_two_new_artifacts(self): pool = morphlib.sourcepool.SourcePool() morph = get_chunk_morphology('chunk', ['chunk-foo', 'chunk-bar']) - chunk = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') - pool.add(chunk) + sources = morphlib.source.make_sources('repo', 'ref', + 'chunk.morph', 'sha1', + 'tree', morph) + for chunk in sources: + pool.add(chunk) morph = get_stratum_morphology( 'stratum', chunks=[ ('chunk', 'chunk', 'repo', 'ref'), ]) - stratum = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'stratum.morph') - pool.add(stratum) + stratum_sources = set(morphlib.source.make_sources('repo', 'ref', + 'stratum.morph', + 'sha1', 'tree', + morph)) + for stratum in stratum_sources: + pool.add(stratum) artifacts = self.resolver.resolve_artifacts(pool) - self.assertEqual(len(artifacts), - sum(len(s.split_rules.artifacts) for s in pool)) + self.assertEqual( + set(artifacts), + set(itertools.chain.from_iterable( + s.artifacts.itervalues() + for s in pool))) - stratum_artifacts = set(a for a in artifacts if a.source == stratum) + stratum_artifacts = set(a for a in artifacts + if a.source in stratum_sources) chunk_artifacts = set(a for a in artifacts if a.source == chunk) for stratum_artifact in stratum_artifacts: self.assertTrue(stratum_artifact.name.startswith('stratum')) self.assertEqual(stratum_artifact.dependents, []) - self.assertTrue(any(dep in chunk_artifacts - for dep in stratum_artifact.dependencies)) + self.assertTrue( + any(dep in chunk_artifacts + for dep in stratum_artifact.source.dependencies)) for chunk_artifact in chunk_artifacts: self.assertTrue(chunk_artifact.name.startswith('chunk')) - self.assertEqual(chunk_artifact.dependencies, []) - self.assertTrue(any(dep in stratum_artifacts + self.assertEqual(chunk_artifact.source.dependencies, []) + self.assertTrue(any(dep in stratum_sources for dep in chunk_artifact.dependents)) - def test_resolving_artifacts_for_a_system_with_two_dependent_strata(self): - pool = morphlib.sourcepool.SourcePool() - - morph = get_chunk_morphology('chunk1') - chunk1 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk1.morph') - pool.add(chunk1) - - morph = get_stratum_morphology( - 'stratum1', - chunks=[('chunk1', 'chunk1', 'repo', 'original/ref')]) - stratum1 = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'stratum1.morph') - pool.add(stratum1) - - loader = morphlib.morphloader.MorphologyLoader() - morph = loader.load_from_string( - ''' - name: system - kind: system - arch: testarch - strata: - - morph: stratum1 - - morph: stratum2 - ''') - morph.builds_artifacts = ['system-rootfs'] - system = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'system.morph') - pool.add(system) - - morph = get_chunk_morphology('chunk2') - chunk2 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk2.morph') - pool.add(chunk2) - - morph = get_stratum_morphology( - 'stratum2', - chunks=[('chunk2', 'chunk2', 'repo', 'original/ref')], - build_depends=['stratum1']) - stratum2 = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'stratum2.morph') - pool.add(stratum2) - - artifacts = self.resolver.resolve_artifacts(pool) - - self.assertEqual(len(artifacts), - sum(len(s.split_rules.artifacts) for s in pool)) - - system_artifacts = set(a for a in artifacts if a.source == system) - stratum1_artifacts = set(a for a in artifacts if a.source == stratum1) - chunk1_artifacts = set(a for a in artifacts if a.source == chunk1) - stratum2_artifacts = set(a for a in artifacts if a.source == stratum2) - chunk2_artifacts = set(a for a in artifacts if a.source == chunk2) - - def assert_depended_on_by_some(artifact, parents): - self.assertNotEqual(len(artifact.dependents), 0) - self.assertTrue(any(a in artifact.dependents for a in parents)) - - def assert_depended_on_by_all(artifact, parents): - self.assertNotEqual(len(artifact.dependents), 0) - self.assertTrue(all(a in artifact.dependents for a in parents)) - - def assert_depends_on_some(artifact, children): - self.assertNotEqual(len(artifact.dependencies), 0) - self.assertTrue(any(a in children for a in artifact.dependencies)) - - def assert_depends_on_all(artifact, children): - self.assertNotEqual(len(artifact.dependencies), 0) - self.assertTrue(all(a in children for a in artifact.dependencies)) - - for c1_a in chunk1_artifacts: - self.assertEqual(c1_a.dependencies, []) - assert_depended_on_by_some(c1_a, stratum1_artifacts) - - for st1_a in stratum1_artifacts: - assert_depends_on_some(st1_a, chunk1_artifacts) - assert_depended_on_by_all(st1_a, chunk2_artifacts) - assert_depended_on_by_some(st1_a, system_artifacts) - - for c2_a in chunk2_artifacts: - assert_depends_on_all(c2_a, stratum1_artifacts) - assert_depended_on_by_some(c2_a, stratum2_artifacts) - - for st2_a in stratum2_artifacts: - assert_depends_on_some(st2_a, chunk2_artifacts) - assert_depended_on_by_some(st2_a, system_artifacts) - - for sy_a in system_artifacts: - self.assertEqual(sy_a.dependents, []) - assert_depends_on_some(sy_a, stratum1_artifacts) - assert_depends_on_some(sy_a, stratum2_artifacts) - - def test_resolving_stratum_with_explicit_chunk_dependencies(self): - pool = morphlib.sourcepool.SourcePool() - - loader = morphlib.morphloader.MorphologyLoader() - morph = loader.load_from_string( - ''' - name: stratum - kind: stratum - build-depends: [] - chunks: - - name: chunk1 - repo: repo - ref: original/ref - build-depends: [] - - name: chunk2 - repo: repo - ref: original/ref - build-depends: [] - - name: chunk3 - repo: repo - ref: original/ref - build-depends: - - chunk1 - - chunk2 - ''') - morph.builds_artifacts = ['stratum'] - stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'stratum.morph') - pool.add(stratum) - - morph = get_chunk_morphology('chunk1') - chunk1 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk1.morph') - pool.add(chunk1) - - morph = get_chunk_morphology('chunk2') - chunk2 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk2.morph') - pool.add(chunk2) - - morph = get_chunk_morphology('chunk3') - chunk3 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk3.morph') - pool.add(chunk3) - - artifacts = self.resolver.resolve_artifacts(pool) - - self.assertEqual(len(artifacts), - sum(len(s.split_rules.artifacts) for s in pool)) - - stratum_artifacts = set(a for a in artifacts if a.source == stratum) - chunk_artifacts = [set(a for a in artifacts if a.source == source) - for source in (chunk1, chunk2, chunk3)] - all_chunks = set(itertools.chain.from_iterable(chunk_artifacts)) - - for st_a in stratum_artifacts: - self.assertEqual(st_a.dependents, []) - # This stratum depends on some chunk artifacts - self.assertTrue(any(a in st_a.dependencies for a in all_chunks)) - - for ca in chunk_artifacts[2]: - # There's a stratum dependent on this artifact - self.assertTrue(any(a in stratum_artifacts for a in ca.dependents)) - # chunk3's artifacts depend on chunk1 and chunk2's artifacts - self.assertEqual(set(ca.dependencies), - chunk_artifacts[0] | chunk_artifacts[1]) - - for ca in itertools.chain.from_iterable(chunk_artifacts[0:1]): - self.assertEqual(ca.dependencies, []) - # There's a stratum dependent on this artifact - self.assertTrue(any(a in stratum_artifacts for a in ca.dependents)) - # All chunk3's artifacts depend on this artifact - self.assertTrue(all(c3a in ca.dependents - for c3a in chunk_artifacts[2])) - def test_detection_of_mutual_dependency_between_two_strata(self): loader = morphlib.morphloader.MorphologyLoader() pool = morphlib.sourcepool.SourcePool() chunk = get_chunk_morphology('chunk1') - chunk1 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', chunk, 'chunk1.morph') + chunk1, = morphlib.source.make_sources( + 'repo', 'original/ref', 'chunk1.morph', 'sha1', 'tree', chunk) pool.add(chunk1) morph = get_stratum_morphology( @@ -398,13 +255,15 @@ class ArtifactResolverTests(unittest.TestCase): chunks=[(loader.save_to_string(chunk), 'chunk1.morph', 'repo', 'original/ref')], build_depends=['stratum2']) - stratum1 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'stratum1.morph') - pool.add(stratum1) + sources = morphlib.source.make_sources('repo', 'original/ref', + 'stratum1.morph', 'sha1', + 'tree', morph) + for stratum1 in sources: + pool.add(stratum1) chunk = get_chunk_morphology('chunk2') - chunk2 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', chunk, 'chunk2.morph') + chunk2, = morphlib.source.make_sources( + 'repo', 'original/ref', 'chunk2.morph', 'sha1', 'tree', chunk) pool.add(chunk2) morph = get_stratum_morphology( @@ -412,9 +271,11 @@ class ArtifactResolverTests(unittest.TestCase): chunks=[(loader.save_to_string(chunk), 'chunk2.morph', 'repo', 'original/ref')], build_depends=['stratum1']) - stratum2 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'stratum2.morph') - pool.add(stratum2) + sources = morphlib.source.make_sources('repo', 'original/ref', + 'stratum2.morph', 'sha1', + 'tree', morph) + for stratum2 in sources: + pool.add(stratum2) self.assertRaises(morphlib.artifactresolver.MutualDependencyError, self.resolver.resolve_artifacts, pool) @@ -439,52 +300,29 @@ class ArtifactResolverTests(unittest.TestCase): ref: original/ref build-depends: [] ''') - morph.builds_artifacts = ['stratum'] - stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'stratum.morph') - pool.add(stratum) + sources = morphlib.source.make_sources('repo', 'original/ref', + 'stratum.morph', 'sha1', + 'tree', morph) + for stratum in sources: + pool.add(stratum) morph = get_chunk_morphology('chunk1') - chunk1 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk1.morph') - pool.add(chunk1) + sources = morphlib.source.make_sources('repo', 'original/ref', + 'chunk1.morph', 'sha1', + 'tree', morph) + for chunk1 in sources: + pool.add(chunk1) morph = get_chunk_morphology('chunk2') - chunk2 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk2.morph') - pool.add(chunk2) + sources = morphlib.source.make_sources('repo', 'original/ref', + 'chunk2.morph', 'sha1', + 'tree', morph) + for chunk2 in sources: + pool.add(chunk2) self.assertRaises(morphlib.artifactresolver.DependencyOrderError, self.resolver.resolve_artifacts, pool) - def test_detection_of_invalid_build_depends_format(self): - pool = morphlib.sourcepool.SourcePool() - - loader = morphlib.morphloader.MorphologyLoader() - morph = loader.load_from_string( - ''' - name: stratum - kind: stratum - build-depends: [] - chunks: - - name: chunk - repo: repo - ref: original/ref - build-depends: whatever - ''') - morph.builds_artifacts = ['stratum'] - stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'stratum.morph') - pool.add(stratum) - - morph = get_chunk_morphology('chunk') - chunk = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk.morph') - pool.add(chunk) - - self.assertRaises(morphlib.artifactresolver.DependencyFormatError, - self.resolver.resolve_artifacts, pool) - # TODO: Expand test suite to include better dependency checking, many # tests were removed due to the fundamental change in how artifacts diff --git a/morphlib/artifactsplitrule.py b/morphlib/artifactsplitrule.py index 125f5b93..cf0d1060 100644 --- a/morphlib/artifactsplitrule.py +++ b/morphlib/artifactsplitrule.py @@ -51,6 +51,9 @@ class FileMatch(Rule): def match(self, path): return any(r.match(path) for r in self._regexes) + def __repr__(self): + return 'FileMatch(%s)' % '|'.join(r.pattern for r in self._regexes) + class ArtifactMatch(Rule): '''Match an artifact's name against a list of regular expressions. @@ -63,6 +66,9 @@ class ArtifactMatch(Rule): def match(self, (source_name, artifact_name)): return any(r.match(artifact_name) for r in self._regexes) + def __repr__(self): + return 'ArtifactMatch(%s)' % '|'.join(r.pattern for r in self._regexes) + class ArtifactAssign(Rule): '''Match only artifacts with the specified source and artifact names. @@ -80,6 +86,9 @@ class ArtifactAssign(Rule): def match(self, (source_name, artifact_name)): return (source_name, artifact_name) == self._key + def __repr__(self): + return 'ArtifactAssign(%s, %s)' % self._key + class SourceAssign(Rule): '''Match only artifacts which come from the specified source. @@ -96,6 +105,9 @@ class SourceAssign(Rule): def match(self, (source_name, artifact_name)): return source_name == self._source + def __repr__(self): + return 'SourceAssign(%s, *)' % self._source + class SplitRules(collections.Iterable): '''Rules engine for splitting a source's artifacts. @@ -172,6 +184,11 @@ class SplitRules(collections.Iterable): return matches, overlaps, unmatched + def __repr__(self): + return 'SplitRules(%s)' % ', '.join( + '%s=%s' % (artifact, rule) + for artifact, rule in self._rules) + # TODO: Work out a good way to feed new defaults in. This is good for # the usual Linux userspace, but we may find issues and need a diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 1cb0c4d9..edd2f0c5 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -14,6 +14,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import itertools import os import shutil import logging @@ -175,9 +176,9 @@ class BuildCommand(object): self.app.status(msg='Computing cache keys', chatty=True) ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) - for artifact in artifacts: - artifact.cache_key = ckc.compute_key(artifact) - artifact.cache_id = ckc.get_cache_id(artifact) + for source in set(a.source for a in artifacts): + source.cache_key = ckc.compute_key(source) + source.cache_id = ckc.get_cache_id(source) root_artifact.build_env = build_env return root_artifact @@ -208,9 +209,9 @@ class BuildCommand(object): # a build-dependency, then they must both have the same Repository # and Ref specified. if src.morphology['kind'] == 'stratum': - name = src.morphology['name'] + name = src.name ref = src.sha1[:7] - self.app.status(msg='Stratum [%(name)s] version is %(ref)s', + self.app.status(msg='Stratum [%(name)s] version is %(ref)s', name=name, ref=ref) if name in stratum_names: raise morphlib.Error( @@ -236,93 +237,109 @@ class BuildCommand(object): logging.debug( 'Validating cross ref to %s:%s:%s' % (repo_name, ref, filename)) - other = srcpool.lookup(repo_name, ref, filename) - if other.morphology['kind'] != wanted: - raise morphlib.Error( - '%s %s references %s:%s:%s which is a %s, ' - 'instead of a %s' % - (src.morphology['kind'], - src.morphology['name'], - repo_name, - ref, - filename, - other.morphology['kind'], - wanted)) + for other in srcpool.lookup(repo_name, ref, filename): + if other.morphology['kind'] != wanted: + raise morphlib.Error( + '%s %s references %s:%s:%s which is a %s, ' + 'instead of a %s' % + (src.morphology['kind'], + src.name, + repo_name, + ref, + filename, + other.morphology['kind'], + wanted)) def _find_root_artifacts(self, artifacts): '''Find all the root artifacts among a set of artifacts in a DAG. - + It would be nice if the ArtifactResolver would return its results in a more useful order to save us from needing to do this -- the root object is known already since that's the one the user asked us to build. - + ''' return [a for a in artifacts if not a.dependents] + @staticmethod + def get_ordered_sources(artifacts): + ordered_sources = [] + known_sources = set() + for artifact in artifacts: + if artifact.source not in known_sources: + known_sources.add(artifact.source) + yield artifact.source + def build_in_order(self, root_artifact): '''Build everything specified in a build order.''' - self.app.status(msg='Building a set of artifacts', chatty=True) + self.app.status(msg='Building a set of sources', chatty=True) build_env = root_artifact.build_env - artifacts = root_artifact.walk() + ordered_sources = list(self.get_ordered_sources(root_artifact.walk())) old_prefix = self.app.status_prefix - for i, a in enumerate(artifacts): + for i, s in enumerate(ordered_sources): self.app.status_prefix = ( old_prefix + '[Build %(index)d/%(total)d] [%(name)s] ' % { 'index': (i+1), - 'total': len(artifacts), - 'name': a.name, + 'total': len(ordered_sources), + 'name': s.name, }) - self.cache_or_build_artifact(a, build_env) - - self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s', - kind=a.source.morphology['kind'], name=a.name, - cachepath=self.lac.artifact_filename(a), - chatty=(a.source.morphology['kind'] != "system")) + self.cache_or_build_source(s, build_env) self.app.status_prefix = old_prefix - def cache_or_build_artifact(self, artifact, build_env): - '''Make the built artifact available in the local cache. + def cache_or_build_source(self, source, build_env): + '''Make artifacts of the built source available in the local cache. This can be done by retrieving from a remote artifact cache, or if - that doesn't work for some reason, by building the artifact locally. + that doesn't work for some reason, by building the source locally. ''' + artifacts = source.artifacts.values() if self.rac is not None: try: - self.cache_artifacts_locally([artifact]) + self.cache_artifacts_locally(artifacts) except morphlib.remoteartifactcache.GetError: # Error is logged by the RemoteArtifactCache object. pass - if not self.lac.has(artifact): - self.build_artifact(artifact, build_env) + if any(not self.lac.has(artifact) for artifact in artifacts): + self.build_source(source, build_env) + + for a in artifacts: + self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s', + kind=source.morphology['kind'], name=a.name, + cachepath=self.lac.artifact_filename(a), + chatty=(source.morphology['kind'] != "system")) - def build_artifact(self, artifact, build_env): - '''Build one artifact. + def build_source(self, source, build_env): + '''Build all artifacts for one source. All the dependencies are assumed to be built and available in either the local or remote cache already. ''' self.app.status(msg='Building %(kind)s %(name)s', - name=artifact.name, - kind=artifact.source.morphology['kind']) - - self.get_sources(artifact) - deps = self.get_recursive_deps(artifact) + name=source.name, + kind=source.morphology['kind']) + + self.fetch_sources(source) + # TODO: Make an artifact.walk() that takes multiple root artifacts. + # as this does a walk for every artifact. This was the status + # quo before build logic was made to work per-source, but we can + # now do better. + deps = self.get_recursive_deps(source.artifacts.values()) self.cache_artifacts_locally(deps) use_chroot = False setup_mounts = False - if artifact.source.morphology['kind'] == 'chunk': - build_mode = artifact.source.build_mode - extra_env = {'PREFIX': artifact.source.prefix} + if source.morphology['kind'] == 'chunk': + build_mode = source.build_mode + extra_env = {'PREFIX': source.prefix} - dep_prefix_set = artifact.get_dependency_prefix_set() + dep_prefix_set = set(a.source.prefix for a in deps + if a.source.morphology['kind'] == 'chunk') extra_path = [os.path.join(d, 'bin') for d in dep_prefix_set] if build_mode not in ['bootstrap', 'staging', 'test']: @@ -340,37 +357,44 @@ class BuildCommand(object): extra_env=extra_env, extra_path=extra_path) try: - self.install_dependencies(staging_area, deps, artifact) + self.install_dependencies(staging_area, deps, source) except BaseException: staging_area.abort() raise else: staging_area = self.create_staging_area(build_env, False) - self.build_and_cache(staging_area, artifact, setup_mounts) + self.build_and_cache(staging_area, source, setup_mounts) self.remove_staging_area(staging_area) - def get_recursive_deps(self, artifact): - return artifact.walk()[:-1] + def get_recursive_deps(self, artifacts): + deps = set() + ordered_deps = [] + for artifact in artifacts: + for dep in artifact.walk(): + if dep not in deps and dep not in artifacts: + deps.add(dep) + ordered_deps.append(dep) + return ordered_deps - def get_sources(self, artifact): + def fetch_sources(self, source): '''Update the local git repository cache with the sources.''' - repo_name = artifact.source.repo_name + repo_name = source.repo_name if self.app.settings['no-git-update']: self.app.status(msg='Not updating existing git repository ' '%(repo_name)s ' 'because of no-git-update being set', chatty=True, repo_name=repo_name) - artifact.source.repo = self.lrc.get_repo(repo_name) + source.repo = self.lrc.get_repo(repo_name) return if self.lrc.has_repo(repo_name): - artifact.source.repo = self.lrc.get_repo(repo_name) + source.repo = self.lrc.get_repo(repo_name) try: - sha1 = artifact.source.sha1 - artifact.source.repo.resolve_ref(sha1) + sha1 = source.sha1 + source.repo.resolve_ref(sha1) self.app.status(msg='Not updating git repository ' '%(repo_name)s because it ' 'already contains sha1 %(sha1)s', @@ -379,17 +403,17 @@ class BuildCommand(object): except morphlib.cachedrepo.InvalidReferenceError: self.app.status(msg='Updating %(repo_name)s', repo_name=repo_name) - artifact.source.repo.update() + source.repo.update() else: self.app.status(msg='Cloning %(repo_name)s', repo_name=repo_name) - artifact.source.repo = self.lrc.cache_repo(repo_name) + source.repo = self.lrc.cache_repo(repo_name) # Update submodules. done = set() self.app.cache_repo_and_submodules( - self.lrc, artifact.source.repo.url, - artifact.source.sha1, done) + self.lrc, source.repo.url, + source.sha1, done) def cache_artifacts_locally(self, artifacts): '''Get artifacts missing from local cache from remote cache.''' @@ -455,14 +479,25 @@ class BuildCommand(object): # Nasty hack to avoid installing chunks built in 'bootstrap' mode in a # different stratum when constructing staging areas. - def is_stratum(self, a): - return a.source.morphology['kind'] == 'stratum' + # TODO: make nicer by having chunk morphs keep a reference to the + # stratum they were in + def in_same_stratum(self, s1, s2): + '''Checks whether two chunk sources are from the same stratum. - def in_same_stratum(self, a, b): - return len(filter(self.is_stratum, a.dependencies)) == \ - len(filter(self.is_stratum, b.dependencies)) + In the absence of morphologies tracking where they came from, + this checks whether both sources are depended on by artifacts + that belong to sources which have the same morphology. - def install_dependencies(self, staging_area, artifacts, target_artifact): + ''' + def dependent_stratum_morphs(source): + dependents = set(itertools.chain.from_iterable( + a.dependents for a in source.artifacts.itervalues())) + dependent_strata = set(s for s in dependents + if s.morphology['kind'] == 'stratum') + return set(s.morphology for s in dependent_strata) + return dependent_stratum_morphs(s1) == dependent_stratum_morphs(s2) + + def install_dependencies(self, staging_area, artifacts, target_source): '''Install chunk artifacts into staging area. We only ever care about chunk artifacts as build dependencies, @@ -477,29 +512,29 @@ class BuildCommand(object): if artifact.source.morphology['kind'] != 'chunk': continue if artifact.source.build_mode == 'bootstrap': - if not self.in_same_stratum(artifact, target_artifact): + if not self.in_same_stratum(artifact.source, target_source): continue self.app.status( msg='Installing chunk %(chunk_name)s from cache %(cache)s', chunk_name=artifact.name, - cache=artifact.cache_key[:7], + cache=artifact.source.cache_key[:7], chatty=True) handle = self.lac.get(artifact) staging_area.install_artifact(handle) - if target_artifact.source.build_mode == 'staging': + if target_source.build_mode == 'staging': morphlib.builder2.ldconfig(self.app.runcmd, staging_area.dirname) - def build_and_cache(self, staging_area, artifact, setup_mounts): - '''Build an artifact and put it into the local artifact cache.''' + def build_and_cache(self, staging_area, source, setup_mounts): + '''Build a source and put its artifacts into the local cache.''' self.app.status(msg='Starting actual build: %(name)s ' '%(sha1)s', - name=artifact.name, sha1=artifact.source.sha1[:7]) + name=source.name, sha1=source.sha1[:7]) builder = morphlib.builder2.Builder( self.app, staging_area, self.lac, self.rac, self.lrc, self.app.settings['max-jobs'], setup_mounts) - return builder.build_and_cache(artifact) + return builder.build_and_cache(source) class InitiatorBuildCommand(BuildCommand): diff --git a/morphlib/builder2.py b/morphlib/builder2.py index c1a49221..20cae225 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -162,15 +162,15 @@ def get_stratum_files(f, lac): # pragma: no cover cf.close() -def get_overlaps(artifact, constituents, lac): # pragma: no cover +def get_overlaps(source, constituents, lac): # pragma: no cover # check whether strata overlap installed = defaultdict(set) for dep in constituents: handle = lac.get(dep) - if artifact.source.morphology['kind'] == 'stratum': + if source.morphology['kind'] == 'stratum': for filename in get_chunk_files(handle): installed[filename].add(dep) - elif artifact.source.morphology['kind'] == 'system': + elif source.morphology['kind'] == 'system': for filename in get_stratum_files(handle, lac): installed[filename].add(dep) handle.close() @@ -207,13 +207,13 @@ class BuilderBase(object): '''Base class for building artifacts.''' def __init__(self, app, staging_area, local_artifact_cache, - remote_artifact_cache, artifact, repo_cache, max_jobs, + remote_artifact_cache, source, repo_cache, max_jobs, setup_mounts): self.app = app self.staging_area = staging_area self.local_artifact_cache = local_artifact_cache self.remote_artifact_cache = remote_artifact_cache - self.artifact = artifact + self.source = source self.repo_cache = repo_cache self.max_jobs = max_jobs self.build_watch = morphlib.stopwatch.Stopwatch() @@ -233,13 +233,13 @@ class BuilderBase(object): logging.debug('Writing metadata to the cache') with self.local_artifact_cache.put_source_metadata( - self.artifact.source, self.artifact.cache_key, + self.source, self.source.cache_key, 'meta') as f: json.dump(meta, f, indent=4, sort_keys=True, encoding='unicode-escape') f.write('\n') - def create_metadata(self, artifact_name, contents=[]): + def create_metadata(self, artifact_name, contents=[]): # pragma: no cover '''Create metadata to artifact to allow it to be reproduced later. The metadata is represented as a dict, which later on will be @@ -247,20 +247,20 @@ class BuilderBase(object): ''' - assert isinstance(self.artifact.source.repo, + assert isinstance(self.source.repo, morphlib.cachedrepo.CachedRepo) meta = { 'artifact-name': artifact_name, - 'source-name': self.artifact.source.morphology['name'], - 'kind': self.artifact.source.morphology['kind'], - 'description': self.artifact.source.morphology['description'], - 'repo': self.artifact.source.repo.url, - 'repo-alias': self.artifact.source.repo_name, - 'original_ref': self.artifact.source.original_ref, - 'sha1': self.artifact.source.sha1, - 'morphology': self.artifact.source.filename, - 'cache-key': self.artifact.cache_key, - 'cache-id': self.artifact.cache_id, + 'source-name': self.source.name, + 'kind': self.source.morphology['kind'], + 'description': self.source.morphology['description'], + 'repo': self.source.repo.url, + 'repo-alias': self.source.repo_name, + 'original_ref': self.source.original_ref, + 'sha1': self.source.sha1, + 'morphology': self.source.filename, + 'cache-key': self.source.cache_key, + 'cache-id': self.source.cache_id, 'morph-version': { 'ref': morphlib.gitversion.ref, 'tree': morphlib.gitversion.tree, @@ -279,7 +279,8 @@ class BuilderBase(object): os.makedirs(dirname) return open(filename, mode) - def write_metadata(self, instdir, artifact_name, contents=[]): + def write_metadata(self, instdir, artifact_name, + contents=[]): # pragma: no cover '''Write the metadata for an artifact. The file will be located under the ``baserock`` directory under @@ -299,12 +300,6 @@ class BuilderBase(object): json.dump(meta, f, indent=4, sort_keys=True, encoding='unicode-escape') f.close() - def new_artifact(self, artifact_name): - '''Return an Artifact object for something built from our source.''' - a = morphlib.artifact.Artifact(self.artifact.source, artifact_name) - a.cache_key = self.artifact.cache_key - return a - def runcmd(self, *args, **kwargs): return self.staging_area.runcmd(*args, **kwargs) @@ -314,7 +309,7 @@ class ChunkBuilder(BuilderBase): def create_devices(self, destdir): # pragma: no cover '''Creates device nodes if the morphology specifies them''' - morphology = self.artifact.source.morphology + morphology = self.source.morphology perms_mask = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO if 'devices' in morphology and morphology['devices'] is not None: for dev in morphology['devices']: @@ -338,14 +333,14 @@ class ChunkBuilder(BuilderBase): with self.build_watch('overall-build'): builddir, destdir = self.staging_area.chroot_open( - self.artifact.source, self.setup_mounts) + self.source, self.setup_mounts) stdout = (self.app.output if self.app.settings['build-log-on-stdout'] else None) cache = self.local_artifact_cache logpath = cache.get_source_metadata_filename( - self.artifact.source, self.artifact.cache_key, 'build-log') + self.source, self.source.cache_key, 'build-log') _, temppath = tempfile.mkstemp(dir=os.path.dirname(logpath)) @@ -381,7 +376,7 @@ class ChunkBuilder(BuilderBase): def run_commands(self, builddir, destdir, logfilepath, stdout=None): # pragma: no cover - m = self.artifact.source.morphology + m = self.source.morphology bs = morphlib.buildsystem.lookup_build_system(m['build-system']) relative_builddir = self.staging_area.relative(builddir) @@ -413,7 +408,7 @@ class ChunkBuilder(BuilderBase): for cmd in cmds: if in_parallel: - max_jobs = self.artifact.source.morphology['max-jobs'] + max_jobs = self.source.morphology['max-jobs'] if max_jobs is None: max_jobs = self.max_jobs extra_env['MAKEFLAGS'] = '-j%s' % max_jobs @@ -480,7 +475,7 @@ class ChunkBuilder(BuilderBase): def assemble_chunk_artifacts(self, destdir): # pragma: no cover built_artifacts = [] filenames = [] - source = self.artifact.source + source = self.source split_rules = source.split_rules morphology = source.morphology sys_tag = 'system-integration' @@ -539,7 +534,7 @@ class ChunkBuilder(BuilderBase): return built_artifacts def get_sources(self, srcdir): # pragma: no cover - s = self.artifact.source + s = self.source extract_sources(self.app, self.repo_cache, s.repo, s.sha1, srcdir) @@ -553,42 +548,42 @@ class StratumBuilder(BuilderBase): def build_and_cache(self): # pragma: no cover with self.build_watch('overall-build'): - constituents = [d for d in self.artifact.dependencies + constituents = [d for d in self.source.dependencies if self.is_constituent(d)] # the only reason the StratumBuilder has to download chunks is to # check for overlap now that strata are lists of chunks with self.build_watch('check-chunks'): - # download the chunk artifact if necessary - download_depends(constituents, - self.local_artifact_cache, - self.remote_artifact_cache) - # check for chunk overlaps - overlaps = get_overlaps(self.artifact, constituents, - self.local_artifact_cache) - if len(overlaps) > 0: - logging.warning('Overlaps in stratum artifact %s detected' - % self.artifact.name) - log_overlaps(overlaps) - self.app.status(msg='Overlaps in stratum artifact ' - '%(stratum_name)s detected', - stratum_name=self.artifact.name, - error=True) - write_overlap_metadata(self.artifact, overlaps, - self.local_artifact_cache) + for a_name, a in self.source.artifacts.iteritems(): + # download the chunk artifact if necessary + download_depends(constituents, + self.local_artifact_cache, + self.remote_artifact_cache) + # check for chunk overlaps + overlaps = get_overlaps(self.source, constituents, + self.local_artifact_cache) + if len(overlaps) > 0: + logging.warning( + 'Overlaps in stratum artifact %s detected' %a_name) + log_overlaps(overlaps) + self.app.status(msg='Overlaps in stratum artifact ' + '%(stratum_name)s detected', + stratum_name=a_name, error=True) + write_overlap_metadata(a, overlaps, + self.local_artifact_cache) with self.build_watch('create-chunk-list'): lac = self.local_artifact_cache - meta = self.create_metadata(self.artifact.name, - [x.name for x in constituents]) - with lac.put_artifact_metadata(self.artifact, 'meta') as f: - json.dump(meta, f, indent=4, sort_keys=True, - encoding='unicode-escape') - with self.local_artifact_cache.put(self.artifact) as f: - json.dump([c.basename() for c in constituents], f, - encoding='unicode-escape') + for a_name, a in self.source.artifacts.iteritems(): + meta = self.create_metadata( + a_name, + [x.name for x in constituents]) + with lac.put_artifact_metadata(a, 'meta') as f: + json.dump(meta, f, indent=4, sort_keys=True) + with self.local_artifact_cache.put(a) as f: + json.dump([c.basename() for c in constituents], f) self.save_build_times() - return [self.artifact] + return self.source.artifacts.values() class SystemBuilder(BuilderBase): # pragma: no cover @@ -602,45 +597,42 @@ class SystemBuilder(BuilderBase): # pragma: no cover def build_and_cache(self): self.app.status(msg='Building system %(system_name)s', - system_name=self.artifact.source.morphology['name']) + system_name=self.source.name) with self.build_watch('overall-build'): - arch = self.artifact.source.morphology['arch'] - - rootfs_name = self.artifact.source.morphology['name'] + '-rootfs' - rootfs_artifact = self.new_artifact(rootfs_name) - handle = self.local_artifact_cache.put(rootfs_artifact) - - try: - fs_root = self.staging_area.destdir(self.artifact.source) - self.unpack_strata(fs_root) - self.write_metadata(fs_root, rootfs_name) - self.run_system_integration_commands(fs_root) - self.copy_kernel_into_artifact_cache(fs_root) - unslashy_root = fs_root[1:] - def uproot_info(info): - info.name = relpath(info.name, unslashy_root) - if info.islnk(): - info.linkname = relpath(info.linkname, - unslashy_root) - return info - artiname = self.artifact.source.morphology['name'] - tar = tarfile.open(fileobj=handle, mode="w", name=artiname) - self.app.status(msg='Constructing tarball of root filesystem', - chatty=True) - tar.add(fs_root, recursive=True, filter=uproot_info) - tar.close() - except BaseException, e: - logging.error(traceback.format_exc()) - self.app.status(msg='Error while building system', - error=True) - handle.abort() - raise - - handle.close() + arch = self.source.morphology['arch'] + + for a_name, artifact in self.source.artifacts.iteritems(): + handle = self.local_artifact_cache.put(artifact) + + try: + fs_root = self.staging_area.destdir(self.source) + self.unpack_strata(fs_root) + self.write_metadata(fs_root, a_name) + self.run_system_integration_commands(fs_root) + unslashy_root = fs_root[1:] + def uproot_info(info): + info.name = relpath(info.name, unslashy_root) + if info.islnk(): + info.linkname = relpath(info.linkname, + unslashy_root) + return info + tar = tarfile.open(fileobj=handle, mode="w", name=a_name) + self.app.status(msg='Constructing tarball of rootfs', + chatty=True) + tar.add(fs_root, recursive=True, filter=uproot_info) + tar.close() + except BaseException as e: + logging.error(traceback.format_exc()) + self.app.status(msg='Error while building system', + error=True) + handle.abort() + raise + else: + handle.close() self.save_build_times() - return [self.artifact] + return self.source.artifacts.itervalues() def unpack_one_stratum(self, stratum_artifact, target): '''Unpack a single stratum into a target directory''' @@ -666,37 +658,37 @@ class SystemBuilder(BuilderBase): # pragma: no cover self.app.status(msg='Unpacking strata to %(path)s', path=path, chatty=True) with self.build_watch('unpack-strata'): - # download the stratum artifacts if necessary - download_depends(self.artifact.dependencies, - self.local_artifact_cache, - self.remote_artifact_cache, - ('meta',)) - - # download the chunk artifacts if necessary - for stratum_artifact in self.artifact.dependencies: - f = self.local_artifact_cache.get(stratum_artifact) - chunks = [ArtifactCacheReference(a) - for a in json.load(f, encoding='unicode-escape')] - download_depends(chunks, + for a_name, a in self.source.artifacts.iteritems(): + # download the stratum artifacts if necessary + download_depends(self.source.dependencies, self.local_artifact_cache, - self.remote_artifact_cache) - f.close() - - # check whether the strata overlap - overlaps = get_overlaps(self.artifact, self.artifact.dependencies, - self.local_artifact_cache) - if len(overlaps) > 0: - self.app.status(msg='Overlaps in system artifact ' - '%(artifact_name)s detected', - artifact_name=self.artifact.name, - error=True) - log_overlaps(overlaps) - write_overlap_metadata(self.artifact, overlaps, - self.local_artifact_cache) - - # unpack it from the local artifact cache - for stratum_artifact in self.artifact.dependencies: - self.unpack_one_stratum(stratum_artifact, path) + self.remote_artifact_cache, + ('meta',)) + + # download the chunk artifacts if necessary + for stratum_artifact in self.source.dependencies: + f = self.local_artifact_cache.get(stratum_artifact) + chunks = [ArtifactCacheReference(c) for c in json.load(f)] + download_depends(chunks, + self.local_artifact_cache, + self.remote_artifact_cache) + f.close() + + # check whether the strata overlap + overlaps = get_overlaps(self.source, self.source.dependencies, + self.local_artifact_cache) + if len(overlaps) > 0: + self.app.status(msg='Overlaps in system artifact ' + '%(artifact_name)s detected', + artifact_name=a_name, + error=True) + log_overlaps(overlaps) + write_overlap_metadata(a, overlaps, + self.local_artifact_cache) + + # unpack it from the local artifact cache + for stratum_artifact in self.source.dependencies: + self.unpack_one_stratum(stratum_artifact, path) ldconfig(self.app.runcmd, path) @@ -766,25 +758,6 @@ class SystemBuilder(BuilderBase): # pragma: no cover % mount_path) morphlib.fsutils.unmount(self.app.runcmd, mount_path) - def copy_kernel_into_artifact_cache(self, path): - '''Copy the installed kernel image into the local artifact cache. - - The kernel image will be a separate artifact from the root - filesystem/disk image/whatever. This is sometimes useful with - funky bootloaders or virtualisation. - - ''' - - name = self.artifact.source.morphology['name'] + '-kernel' - a = self.new_artifact(name) - with self.local_artifact_cache.put(a) as dest: - for basename in ['zImage', 'vmlinuz']: - installed_path = os.path.join(path, 'boot', basename) - if os.path.exists(installed_path): - with open(installed_path) as kernel: - shutil.copyfileobj(kernel, dest) - break - class Builder(object): # pragma: no cover @@ -806,15 +779,15 @@ class Builder(object): # pragma: no cover self.max_jobs = max_jobs self.setup_mounts = setup_mounts - def build_and_cache(self, artifact): - kind = artifact.source.morphology['kind'] + def build_and_cache(self, source): + kind = source.morphology['kind'] o = self.classes[kind](self.app, self.staging_area, self.local_artifact_cache, - self.remote_artifact_cache, artifact, + self.remote_artifact_cache, source, self.repo_cache, self.max_jobs, self.setup_mounts) self.app.status(msg='Builder.build: artifact %s with %s' % - (artifact.name, repr(o)), + (source.name, repr(o)), chatty=True) built_artifacts = o.build_and_cache() self.app.status(msg='Builder.build: done', diff --git a/morphlib/builder2_tests.py b/morphlib/builder2_tests.py index d0d56b17..4fd0807a 100644 --- a/morphlib/builder2_tests.py +++ b/morphlib/builder2_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -48,6 +48,7 @@ class FakeSource(object): 'kind': 'b', 'description': 'c', } + self.name = 'a' self.repo = morphlib.cachedrepo.CachedRepo(FakeApp(), 'repo', 'url', 'path') @@ -162,45 +163,10 @@ class BuilderBaseTests(unittest.TestCase): self.max_jobs, False) - def test_returns_an_artifact(self): - artifact = self.builder.new_artifact('le-artifact') - self.assertEqual(type(artifact), morphlib.artifact.Artifact) - def test_runs_desired_command(self): self.builder.runcmd(['foo', 'bar']) self.assertEqual(self.commands_run, [['foo', 'bar']]) - def test_creates_metadata_with_required_fields(self): - artifact_name = 'le-artifact' - source = self.artifact.source - morphology = source.morphology - meta = self.builder.create_metadata(artifact_name) - self.assertEqual(meta['artifact-name'], artifact_name) - self.assertEqual(meta['source-name'], morphology['name']) - self.assertEqual(meta['kind'], morphology['kind']) - self.assertEqual(meta['description'], morphology['description']) - self.assertEqual(meta['repo'], source.repo.url) - self.assertEqual(meta['original_ref'], source.original_ref) - self.assertEqual(meta['sha1'], source.sha1) - self.assertEqual(meta['morphology'], source.filename) - - def test_writes_metadata(self): - artifact_name = 'le-artifact' - orig_meta = self.builder.create_metadata(artifact_name) - - instdir = '/inst/dir' - - self.builder._open = self.fake_open - self.builder.write_metadata(instdir, artifact_name) - - self.assertTrue(self.open_filename.startswith( - os.path.join(instdir, 'baserock', - artifact_name + '.'))) - self.assertTrue(self.open_filename.endswith('.meta')) - - meta = json.loads(self.open_handle.getvalue()) - self.assertEqual(meta, orig_meta) - def test_writes_build_times(self): with self.builder.build_watch('nothing'): pass diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index cf44f76e..c3a01b9e 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -32,16 +32,15 @@ class CacheKeyComputer(object): "USER", "USERNAME"] return dict([(k, env[k]) for k in keys]) - def compute_key(self, artifact): + def compute_key(self, source): try: - ret = self._hashed[artifact] - return ret + return self._hashed[source] except KeyError: - ret = self._hash_id(self.get_cache_id(artifact)) - self._hashed[artifact] = ret - logging.debug('computed cache key %s for artifact %s from source ', - ret, (artifact.source.repo_name, - artifact.source.sha1, artifact.source.filename)) + ret = self._hash_id(self.get_cache_id(source)) + self._hashed[source] = ret + logging.debug( + 'computed cache key %s for artifact %s from source ', + ret, (source.repo_name, source.sha1, source.filename)) return ret def _hash_id(self, cache_id): @@ -71,47 +70,47 @@ class CacheKeyComputer(object): for item in tup: self._hash_thing(sha, item) - def get_cache_id(self, artifact): + def get_cache_id(self, source): try: - ret = self._calculated[artifact] + ret = self._calculated[source] return ret except KeyError: - cacheid = self._calculate(artifact) - self._calculated[artifact] = cacheid + cacheid = self._calculate(source) + self._calculated[source] = cacheid return cacheid - def _calculate(self, artifact): + def _calculate(self, source): keys = { 'env': self._filterenv(self._build_env.env), - 'kids': [{'artifact': a.name, 'cache-key': self.compute_key(a)} - for a in artifact.dependencies], - 'metadata-version': artifact.metadata_version + 'kids': [{'artifact': a.name, + 'cache-key': self.compute_key(a.source)} + for a in source.dependencies], + 'metadata-version': 1 } - kind = artifact.source.morphology['kind'] + morphology = source.morphology + kind = morphology['kind'] if kind == 'chunk': - keys['build-mode'] = artifact.source.build_mode - keys['prefix'] = artifact.source.prefix - keys['tree'] = artifact.source.tree + keys['build-mode'] = source.build_mode + keys['prefix'] = source.prefix + keys['tree'] = source.tree keys['split-rules'] = [(a, [rgx.pattern for rgx in r._regexes]) - for (a, r) in artifact.source.split_rules] + for (a, r) in source.split_rules] # Include morphology contents, since it doesn't always come # from the source tree - morphology = artifact.source.morphology + keys['devices'] = morphology.get('devices') + keys['max-jobs'] = morphology.get('max-jobs') + keys['system-integration'] = morphology.get('system-integration', + {}) + # products is omitted as they are part of the split-rules # include {pre-,,post-}{configure,build,test,install}-commands # in morphology key for prefix in ('pre-', '', 'post-'): for cmdtype in ('configure', 'build', 'test', 'install'): cmd_field = prefix + cmdtype + '-commands' keys[cmd_field] = morphology[cmd_field] - keys['devices'] = morphology.get('devices') - keys['max-jobs'] = morphology.get('max-jobs') - keys['system-integration'] = morphology.get('system-integration', - {}) - # products is omitted as they are part of the split-rules elif kind in ('system', 'stratum'): - morphology = artifact.source.morphology morph_dict = dict((k, morphology[k]) for k in morphology.keys()) # Disregard all fields of a morphology that aren't important diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py index 8558db6d..55936f94 100644 --- a/morphlib/cachekeycomputer_tests.py +++ b/morphlib/cachekeycomputer_tests.py @@ -83,18 +83,14 @@ class CacheKeyComputerTests(unittest.TestCase): - morph: stratum2 ''', }.iteritems(): - source = morphlib.source.Source( - 'repo', 'original/ref', 'sha', 'tree', - loader.load_from_string(text), name) - self.source_pool.add(source) + morph = loader.load_from_string(text) + sources = morphlib.source.make_sources('repo', 'original/ref', + name, 'sha1', + 'tree', morph) + for source in sources: + self.source_pool.add(source) # FIXME: This should use MorphologyFactory m = source.morphology - if m['kind'] == 'system': - m.builds_artifacts = [m['name'] + '-rootfs'] - elif m['kind'] == 'stratum': - m.builds_artifacts = [m['name']] - elif m['kind'] == 'chunk': - m.builds_artifacts = [m['name']] self.build_env = DummyBuildEnvironment({ "LOGNAME": "foouser", "MORPH_ARCH": "dummy", @@ -127,7 +123,7 @@ class CacheKeyComputerTests(unittest.TestCase): self.ckc._hash_tuple = inccount(self.ckc._hash_tuple, 'tuple') artifact = self._find_artifact('system-rootfs') - self.ckc.compute_key(artifact) + self.ckc.compute_key(artifact.source) self.assertNotEqual(runcount['thing'], 0) self.assertNotEqual(runcount['dict'], 0) @@ -140,13 +136,13 @@ class CacheKeyComputerTests(unittest.TestCase): def test_compute_twice_same_key(self): artifact = self._find_artifact('system-rootfs') - self.assertEqual(self.ckc.compute_key(artifact), - self.ckc.compute_key(artifact)) + self.assertEqual(self.ckc.compute_key(artifact.source), + self.ckc.compute_key(artifact.source)) def test_compute_twice_same_id(self): artifact = self._find_artifact('system-rootfs') - id1 = self.ckc.get_cache_id(artifact) - id2 = self.ckc.get_cache_id(artifact) + id1 = self.ckc.get_cache_id(artifact.source) + id2 = self.ckc.get_cache_id(artifact.source) hash1 = self.ckc._hash_id(id1) hash2 = self.ckc._hash_id(id2) self.assertEqual(hash1, hash2) @@ -154,50 +150,13 @@ class CacheKeyComputerTests(unittest.TestCase): def test_compute_key_returns_sha256(self): artifact = self._find_artifact('system-rootfs') self.assertTrue(self._valid_sha256( - self.ckc.compute_key(artifact))) + self.ckc.compute_key(artifact.source))) def test_different_env_gives_different_key(self): artifact = self._find_artifact('system-rootfs') - oldsha = self.ckc.compute_key(artifact) + oldsha = self.ckc.compute_key(artifact.source) build_env = copy.deepcopy(self.build_env) build_env.env["USER"] = "brian" ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) - self.assertNotEqual(oldsha, ckc.compute_key(artifact)) - - def test_same_morphology_text_but_changed_sha1_gives_same_cache_key(self): - old_artifact = self._find_artifact('system-rootfs') - morphology = old_artifact.source.morphology - new_source = morphlib.source.Source('repo', 'original/ref', 'newsha', - 'tree', morphology, - old_artifact.source.filename) - sp = morphlib.sourcepool.SourcePool() - for source in self.source_pool: - if source == old_artifact.source: - sp.add(new_source) - else: - sp.add(source) - artifacts = self.artifact_resolver.resolve_artifacts(sp) - for new_artifact in artifacts: - if new_artifact.source == new_source: - break - else: - self.assertTrue(False) - - old_sha = self.ckc.compute_key(old_artifact) - new_sha = self.ckc.compute_key(new_artifact) - self.assertEqual(old_sha, new_sha) - - def test_same_morphology_added_to_source_pool_only_appears_once(self): - loader = morphlib.morphloader.MorphologyLoader() - m = loader.load_from_string( - ''' - name: chunk - kind: chunk - ''') - src = morphlib.source.Source('repo', 'original/ref', 'sha', 'tree', m, - 'chunk.morph') - sp = morphlib.sourcepool.SourcePool() - sp.add(src) - sp.add(src) - self.assertEqual(1, len([s for s in sp if s == src])) + self.assertNotEqual(oldsha, ckc.compute_key(artifact.source)) diff --git a/morphlib/localartifactcache_tests.py b/morphlib/localartifactcache_tests.py index 6283c833..4325cfbe 100644 --- a/morphlib/localartifactcache_tests.py +++ b/morphlib/localartifactcache_tests.py @@ -43,14 +43,15 @@ class LocalArtifactCacheTests(unittest.TestCase): include: - usr/include ''') - self.source = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') + sources = morphlib.source.make_sources('repo', 'ref', + 'chunk.morph', 'sha1', + 'tree', morph) + self.source, = sources + self.source.cache_key = '0'*64 self.runtime_artifact = morphlib.artifact.Artifact( self.source, 'chunk-runtime') - self.runtime_artifact.cache_key = '0'*64 self.devel_artifact = morphlib.artifact.Artifact( self.source, 'chunk-devel') - self.devel_artifact.cache_key = '0'*64 def test_artifact_filename(self): cache = morphlib.localartifactcache.LocalArtifactCache(self.tempfs) @@ -61,12 +62,13 @@ class LocalArtifactCacheTests(unittest.TestCase): def test_get_source_metadata_filename(self): cache = morphlib.localartifactcache.LocalArtifactCache(self.tempfs) artifact = self.devel_artifact + source = self.source name = 'foobar' filename = cache.get_source_metadata_filename(artifact.source, - artifact.cache_key, name) + source.cache_key, name) expected_name = self.tempfs.getsyspath('%s.%s' % - (artifact.cache_key, name)) + (source.cache_key, name)) self.assertEqual(filename, expected_name) def test_put_artifacts_and_check_whether_the_cache_has_them(self): @@ -165,13 +167,13 @@ class LocalArtifactCacheTests(unittest.TestCase): handle.write('runtime') handle.close() - self.assertTrue(len(list(cache.list_contents())) == 1) + self.assertEqual(len(list(cache.list_contents())), 1) handle = cache.put(self.devel_artifact) handle.write('devel') handle.close() - self.assertTrue(len(list(cache.list_contents())) == 1) + self.assertEqual(len(list(cache.list_contents())), 1) def test_put_artifacts_and_remove_them_afterwards(self): cache = morphlib.localartifactcache.LocalArtifactCache(self.tempfs) @@ -187,4 +189,4 @@ class LocalArtifactCacheTests(unittest.TestCase): key = list(cache.list_contents())[0][0] cache.remove(key) - self.assertTrue(len(list(cache.list_contents())) == 0) + self.assertEqual(len(list(cache.list_contents())), 0) diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py index bc7ab6ef..8289b01e 100644 --- a/morphlib/morphloader.py +++ b/morphlib/morphloader.py @@ -536,7 +536,12 @@ class MorphologyLoader(object): # Require build-dependencies for the stratum itself, unless # it has chunks built in bootstrap mode. - if 'build-depends' not in morph: + if 'build-depends' in morph: + if not isinstance(morph['build-depends'], list): + raise InvalidTypeError( + 'build-depends', list, type(morph['build-depends']), + morph['name']) + else: for spec in morph['chunks']: if spec.get('build-mode') in ['bootstrap', 'test']: break @@ -549,11 +554,15 @@ class MorphologyLoader(object): # Require build-dependencies for each chunk. for spec in morph['chunks']: - if 'build-depends' not in spec: + chunk_name = spec.get('alias', spec['name']) + if 'build-depends' in spec: + if not isinstance(spec['build-depends'], list): + raise InvalidTypeError( + '%s.build-depends' % chunk_name, list, + type(spec['build-depends']), morph['name']) + else: raise NoBuildDependenciesError( - morph['name'], - spec.get('alias', spec['name']), - morph.filename) + morph['name'], chunk_name, morph.filename) @classmethod def _validate_chunk(cls, morphology): diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py index e783bfdb..dd70c824 100644 --- a/morphlib/morphloader_tests.py +++ b/morphlib/morphloader_tests.py @@ -394,6 +394,48 @@ build-system: dummy m['chunks'][0]['build-mode'] = 'bootstrap' self.loader.validate(m) + def test_validate_stratum_build_deps_are_list(self): + m = morphlib.morphology.Morphology( + { + "name": "stratum-invalid-bdeps", + "kind": "stratum", + "build-depends": 0.1, + "chunks": [ + { + "name": "chunk", + "repo": "test:repo", + "ref": "sha1", + "build-depends": [] + } + ] + }) + + self.assertRaises( + morphlib.morphloader.InvalidTypeError, + self.loader.validate, m) + + def test_validate_chunk_build_deps_are_list(self): + m = morphlib.morphology.Morphology( + { + "name": "stratum-invalid-bdeps", + "kind": "stratum", + "build-depends": [ + { "morph": "foo" }, + ], + "chunks": [ + { + "name": "chunk", + "repo": "test:repo", + "ref": "sha1", + "build-depends": 0.1 + } + ] + }) + + self.assertRaises( + morphlib.morphloader.InvalidTypeError, + self.loader.validate, m) + def test_validate_requires_chunks_in_strata(self): m = morphlib.morphology.Morphology( { diff --git a/morphlib/morphology.py b/morphlib/morphology.py index 314c315a..9bd9bb5b 100644 --- a/morphlib/morphology.py +++ b/morphlib/morphology.py @@ -43,3 +43,9 @@ class Morphology(UserDict.IterableUserDict): self.filename = None self.dirty = None + @property + def needs_artifact_metadata_cached(self): # pragma: no cover + return self.get('kind') == 'stratum' + + def __hash__(self): + return id(self) diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py index 1a8e374e..b0a0528d 100644 --- a/morphlib/morphologyfactory.py +++ b/morphlib/morphologyfactory.py @@ -50,7 +50,7 @@ class MorphologyFactory(object): if self._app is not None: self._app.status(*args, **kwargs) - def _load_morphology(self, reponame, sha1, filename): + def get_morphology(self, reponame, sha1, filename): morph_name = os.path.splitext(os.path.basename(filename))[0] loader = morphlib.morphloader.MorphologyLoader() if self._lrc.has_repo(reponame): @@ -88,38 +88,3 @@ class MorphologyFactory(object): loader.set_commands(morph) loader.set_defaults(morph) return morph - - def get_morphology(self, reponame, sha1, filename): - morphology = self._load_morphology(reponame, sha1, filename) - - method_name = '_check_and_tweak_%s' % morphology['kind'] - if hasattr(self, method_name): - method = getattr(self, method_name) - method(morphology, reponame, sha1, filename) - - return morphology - - def _check_and_tweak_system(self, morphology, reponame, sha1, filename): - '''Check and tweak a system morphology.''' - - name = morphology['name'] - morphology.builds_artifacts = [name + '-rootfs'] - - morphology.needs_artifact_metadata_cached = False - - def _check_and_tweak_stratum(self, morphology, reponame, sha1, filename): - '''Check and tweak a stratum morphology.''' - - morphology.builds_artifacts = [morphology['name']] - morphology.needs_artifact_metadata_cached = True - - def _check_and_tweak_chunk(self, morphology, reponame, sha1, filename): - '''Check and tweak a chunk morphology.''' - - if 'products' in morphology and len(morphology['products']) > 1: - morphology.builds_artifacts = [d['artifact'] - for d in morphology['products']] - else: - morphology.builds_artifacts = [morphology['name']] - - morphology.needs_artifact_metadata_cached = False diff --git a/morphlib/morphologyfactory_tests.py b/morphlib/morphologyfactory_tests.py index 0b3253da..52d5f598 100644 --- a/morphlib/morphologyfactory_tests.py +++ b/morphlib/morphologyfactory_tests.py @@ -248,37 +248,6 @@ class MorphologyFactoryTests(unittest.TestCase): self.assertRaises(NotcachedError, self.lmf.get_morphology, 'reponame', 'sha1', 'unreached.morph') - def test_sets_builds_artifacts_for_simple_chunk(self): - morph = self.mf.get_morphology('reponame', 'sha1', 'chunk.morph') - self.assertEqual(morph.builds_artifacts, ['chunk']) - - def test_sets_builds_artifacts_for_split_chunk(self): - morph = self.mf.get_morphology('reponame', 'sha1', 'chunk-split.morph') - self.assertEqual(morph.builds_artifacts, - ['chunk-split-runtime', 'chunk-split-devel']) - - def test_sets_builds_artifacts_for_stratum(self): - morph = self.mf.get_morphology('reponame', 'sha1', 'stratum.morph') - self.assertEqual(morph.builds_artifacts, ['stratum']) - - def test_sets_build_artifacts_for_system(self): - self.lr.arch = 'x86_32' - morph = self.mf.get_morphology('reponame', 'sha1', 'system.morph') - self.assertEqual(morph.builds_artifacts, ['system-rootfs']) - - def test_does_not_set_needs_artifact_metadata_cached_for_chunk(self): - morph = self.mf.get_morphology('reponame', 'sha1', 'chunk.morph') - self.assertEqual(morph.needs_artifact_metadata_cached, False) - - def test_sets_artifact_metadata_cached_for_stratum(self): - morph = self.mf.get_morphology('reponame', 'sha1', 'stratum.morph') - self.assertEqual(morph.needs_artifact_metadata_cached, True) - - def test_does_not_set_artifact_metadata_cached_for_system(self): - morph = self.mf.get_morphology('reponame', 'sha1', 'system.morph') - self.assertEqual(morph.needs_artifact_metadata_cached, False) - - def test_arch_is_validated(self): self.lr.arch = 'unknown' self.assertRaises(morphlib.Error, self.mf.get_morphology, diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py index 0c3e3a4a..7b53a4a5 100644 --- a/morphlib/plugins/cross-bootstrap_plugin.py +++ b/morphlib/plugins/cross-bootstrap_plugin.py @@ -58,38 +58,39 @@ class BootstrapSystemBuilder(morphlib.builder2.BuilderBase): def build_and_cache(self): with self.build_watch('overall-build'): - handle = self.local_artifact_cache.put(self.artifact) - fs_root = self.staging_area.destdir(self.artifact.source) - try: - self.unpack_binary_chunks(fs_root) - self.unpack_sources(fs_root) - self.write_build_script(fs_root) - system_name = self.artifact.source.morphology['name'] - self.create_tarball(handle, fs_root, system_name) - except BaseException, e: - logging.error(traceback.format_exc()) - self.app.status(msg='Error while building bootstrap image', - error=True) - handle.abort() - raise - - handle.close() + for system_name, artifact in self.source.artifacts.iteritems(): + handle = self.local_artifact_cache.put(artifact) + fs_root = self.staging_area.destdir(self.source) + try: + self.unpack_binary_chunks(fs_root) + self.unpack_sources(fs_root) + self.write_build_script(fs_root) + self.create_tarball(handle, fs_root, system_name) + except BaseException, e: + logging.error(traceback.format_exc()) + self.app.status(msg='Error while building bootstrap image', + error=True) + handle.abort() + raise + + handle.close() self.save_build_times() - return [self.artifact] + return self.source.artifacts.items() def unpack_binary_chunks(self, dest): cache = self.local_artifact_cache - for chunk_artifact in self.artifact.source.cross_chunks: - with cache.get(chunk_artifact) as chunk_file: - try: - morphlib.bins.unpack_binary_from_file(chunk_file, dest) - except BaseException, e: - self.app.status( - msg='Error unpacking binary chunk %(name)s', - name=chunk_artifact.name, - error=True) - raise + for chunk_source in self.source.cross_sources: + for chunk_artifact in chunk_source.artifacts.itervalues(): + with cache.get(chunk_artifact) as chunk_file: + try: + morphlib.bins.unpack_binary_from_file(chunk_file, dest) + except BaseException, e: + self.app.status( + msg='Error unpacking binary chunk %(name)s', + name=chunk_artifact.name, + error=True) + raise def unpack_sources(self, path): # Multiple chunks sources may be built from the same repo ('linux' @@ -98,24 +99,18 @@ class BootstrapSystemBuilder(morphlib.builder2.BuilderBase): # # It might be neater to build these as "source artifacts" individually, # but that would waste huge amounts of space in the artifact cache. - for a in self.artifact.walk(): - if a in self.artifact.source.cross_chunks: - continue - if a.source.morphology['kind'] != 'chunk': - continue - - escaped_source = escape_source_name(a.source) + for s in self.source.native_sources: + escaped_source = escape_source_name(s) source_dir = os.path.join(path, 'src', escaped_source) if not os.path.exists(source_dir): os.makedirs(source_dir) morphlib.builder2.extract_sources( - self.app, self.repo_cache, a.source.repo, a.source.sha1, - source_dir) + self.app, self.repo_cache, s.repo, s.sha1, source_dir) - name = a.source.morphology['name'] + name = s.name chunk_script = os.path.join(path, 'src', 'build-%s' % name) with morphlib.savefile.SaveFile(chunk_script, 'w') as f: - self.write_chunk_build_script(a, f) + self.write_chunk_build_script(s, f) os.chmod(chunk_script, 0777) def write_build_script(self, path): @@ -130,15 +125,8 @@ class BootstrapSystemBuilder(morphlib.builder2.BuilderBase): if k != 'PATH': f.write('export %s="%s"\n' % (k, v)) - # FIXME: really, of course, we need to iterate the sources not the - # artifacts ... this will break when we have chunk splitting! - for a in self.artifact.walk(): - if a in self.artifact.source.cross_chunks: - continue - if a.source.morphology['kind'] != 'chunk': - continue - - name = a.source.morphology['name'] + for s in self.source.native_sources: + name = s.name f.write('\necho Building %s\n' % name) f.write('mkdir /%s.inst\n' % name) f.write('env DESTDIR=/%s.inst $SRCDIR/build-%s\n' @@ -150,17 +138,17 @@ class BootstrapSystemBuilder(morphlib.builder2.BuilderBase): f.write(driver_footer) os.chmod(driver_script, 0777) - def write_chunk_build_script(self, chunk, f): - m = chunk.source.morphology + def write_chunk_build_script(self, source, f): + m = source.morphology f.write('#!/bin/sh\n') f.write('# Build script generated by morph\n') f.write('set -e\n') f.write('chunk_name=%s\n' % m['name']) - repo = escape_source_name(chunk.source) + repo = escape_source_name(source) f.write('cp -a $SRCDIR/%s $DESTDIR/$chunk_name.build\n' % repo) f.write('cd $DESTDIR/$chunk_name.build\n') - f.write('export PREFIX=%s\n' % chunk.source.prefix) + f.write('export PREFIX=%s\n' % source.prefix) bs = morphlib.buildsystem.lookup_build_system(m['build-system']) @@ -261,12 +249,8 @@ class CrossBootstrapPlugin(cliapp.Plugin): build_command = morphlib.buildcommand.BuildCommand(self.app, build_env) morph_name = morphlib.util.sanitise_morphology_path(system_name) - builds_artifacts = [system_name + '-bootstrap-rootfs'] srcpool = build_command.create_source_pool(root_repo, ref, morph_name) - system_source = srcpool.lookup(root_repo, ref, morph_name) - system_source.morphology.builds_artifacts = builds_artifacts - # FIXME: this is a quick fix in order to get it working for # Baserock 13 release, it is not a reasonable fix def validate(self, root_artifact): @@ -284,35 +268,36 @@ class CrossBootstrapPlugin(cliapp.Plugin): # Calculate build order # This is basically a hacked version of BuildCommand.build_in_order() - artifacts = system_artifact.walk() - cross_chunks = [] - native_chunks = [] - for a in artifacts: - if a.source.morphology['kind'] == 'chunk': - if a.source.build_mode == 'bootstrap': - cross_chunks.append(a) + sources = build_command.get_ordered_sources(system_artifact.walk()) + cross_sources = [] + native_sources = [] + for s in sources: + if s.morphology['kind'] == 'chunk': + if s.build_mode == 'bootstrap': + cross_sources.append(s) else: - native_chunks.append(a) + native_sources.append(s) - if len(cross_chunks) == 0: + if len(cross_sources) == 0: raise morphlib.Error( 'Nothing to cross-compile. Only chunks built in \'bootstrap\' ' 'mode can be cross-compiled.') - for i, a in enumerate(cross_chunks): - build_command.cache_or_build_artifact(a, build_env) + for s in cross_sources: + build_command.cache_or_build_source(s, build_env) - for i, a in enumerate(native_chunks): - build_command.get_sources(a) + for s in native_sources: + build_command.fetch_sources(s) # Install those to the output tarball ... self.app.status(msg='Building final bootstrap system image') - system_artifact.source.cross_chunks = cross_chunks + system_artifact.source.cross_sources = cross_sources + system_artifact.source.native_sources = native_sources staging_area = build_command.create_staging_area( build_env, use_chroot=False) builder = BootstrapSystemBuilder( self.app, staging_area, build_command.lac, build_command.rac, - system_artifact, build_command.lrc, 1, False) + system_artifact.source, build_command.lrc, 1, False) builder.build_and_cache() self.app.status( diff --git a/morphlib/plugins/show_dependencies_plugin.py b/morphlib/plugins/show_dependencies_plugin.py index 3a1cb7ad..e70f6bfb 100644 --- a/morphlib/plugins/show_dependencies_plugin.py +++ b/morphlib/plugins/show_dependencies_plugin.py @@ -74,6 +74,6 @@ class ShowDependenciesPlugin(cliapp.Plugin): for artifact in reversed(root_artifact.walk()): self.app.output.write(' %s\n' % artifact) - for dependency in sorted(artifact.dependencies, key=str): - self.app.output.write(' -> %s\n' % dependency) + for dep in sorted(artifact.source.dependencies, key=str): + self.app.output.write(' -> %s\n' % dep) diff --git a/morphlib/remoteartifactcache.py b/morphlib/remoteartifactcache.py index 0f8edce8..4e09ce34 100644 --- a/morphlib/remoteartifactcache.py +++ b/morphlib/remoteartifactcache.py @@ -31,9 +31,9 @@ class GetError(cliapp.AppException): def __init__(self, cache, artifact): cliapp.AppException.__init__( - self, 'Failed to get the artifact %s with cache key %s ' + self, 'Failed to get the artifact %s ' 'from the artifact cache %s' % - (artifact.basename(), artifact.cache_key, cache)) + (artifact.basename(), cache)) class GetArtifactMetadataError(GetError): diff --git a/morphlib/remoteartifactcache_tests.py b/morphlib/remoteartifactcache_tests.py index ca959ebf..788882c2 100644 --- a/morphlib/remoteartifactcache_tests.py +++ b/morphlib/remoteartifactcache_tests.py @@ -43,17 +43,19 @@ class RemoteArtifactCacheTests(unittest.TestCase): include: - usr/share/doc ''') - self.source = morphlib.source.Source( - 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') + sources = morphlib.source.make_sources('repo', 'original/ref', + 'chunk.morph', 'sha1', + 'tree', morph) + self.source, = sources self.runtime_artifact = morphlib.artifact.Artifact( self.source, 'chunk-runtime') - self.runtime_artifact.cache_key = 'CHUNK-RUNTIME' + self.runtime_artifact.cache_key = 'CHUNK' self.devel_artifact = morphlib.artifact.Artifact( self.source, 'chunk-devel') - self.devel_artifact.cache_key = 'CHUNK-DEVEL' + self.devel_artifact.cache_key = 'CHUNK' self.doc_artifact = morphlib.artifact.Artifact( self.source, 'chunk-doc') - self.doc_artifact.cache_key = 'CHUNK-DOC' + self.doc_artifact.cache_key = 'CHUNK' self.existing_files = set([ self.runtime_artifact.basename(), diff --git a/morphlib/source.py b/morphlib/source.py index 2dbabad1..4ad54ed9 100644 --- a/morphlib/source.py +++ b/morphlib/source.py @@ -30,13 +30,17 @@ class Source(object): * ``tree`` -- the SHA1 of the tree corresponding to the commit * ``morphology`` -- the in-memory representation of the morphology we use * ``filename`` -- basename of the morphology filename - * ``artifacts`` -- the set of artifacts this source produces. + * ``cache_id`` -- a dict describing the components of the cache key + * ``cache_key`` -- a cache key to uniquely identify the artifact + * ``dependencies`` -- list of Artifacts that need to be built beforehand * ``split_rules`` -- rules for splitting the source's produced artifacts + * ``artifacts`` -- the set of artifacts this source produces. ''' - def __init__(self, repo_name, original_ref, sha1, tree, morphology, - filename): + def __init__(self, name, repo_name, original_ref, sha1, tree, morphology, + filename, split_rules): + self.name = name self.repo = None self.repo_name = repo_name self.original_ref = original_ref @@ -44,15 +48,63 @@ class Source(object): self.tree = tree self.morphology = morphology self.filename = filename + self.cache_id = None + self.cache_key = None + self.dependencies = [] - kind = morphology['kind'] - unifier = getattr(morphlib.artifactsplitrule, - 'unify_%s_matches' % kind) - self.split_rules = unifier(morphology) - self.artifacts = {name: morphlib.artifact.Artifact(self, name) - for name in self.split_rules.artifacts} + self.split_rules = split_rules + self.artifacts = None def __str__(self): # pragma: no cover - return '%s|%s|%s' % (self.repo_name, - self.original_ref, - self.filename) + return '%s|%s|%s|%s' % (self.repo_name, + self.original_ref, + self.filename, + self.name) + + def __repr__(self): # pragma: no cover + return 'Source(%s)' % str(self) + + def basename(self): # pragma: no cover + return '%s.%s' % (self.cache_key, str(self.morphology['kind'])) + + def add_dependency(self, artifact): # pragma: no cover + if artifact not in self.dependencies: + self.dependencies.append(artifact) + if self not in artifact.dependents: + artifact.dependents.append(self) + + def depends_on(self, artifact): # pragma: no cover + '''Do we depend on ``artifact``?''' + return artifact in self.dependencies + + +def make_sources(reponame, ref, filename, absref, tree, morphology): + kind = morphology['kind'] + if kind in ('system', 'chunk'): + unifier = getattr(morphlib.artifactsplitrule, + 'unify_%s_matches' % kind) + split_rules = unifier(morphology) + # chunk and system sources are named after the morphology + source_name = morphology['name'] + source = morphlib.source.Source(source_name, reponame, ref, + absref, tree, morphology, + filename, split_rules) + source.artifacts = {name: morphlib.artifact.Artifact(source, name) + for name in split_rules.artifacts} + yield source + elif kind == 'stratum': # pragma: no cover + unifier = morphlib.artifactsplitrule.unify_stratum_matches + split_rules = unifier(morphology) + for name in split_rules.artifacts: + source = morphlib.source.Source( + name, # stratum source name is artifact name + reponame, ref, absref, tree, morphology, filename, + # stratum sources need to match the unified + # split rules, so they know to yield the match + # to a different source + split_rules) + source.artifacts = {name: morphlib.artifact.Artifact(source, name)} + yield source + else: + # cluster morphologies don't have sources + pass diff --git a/morphlib/source_tests.py b/morphlib/source_tests.py index f5ce5d4d..695041d3 100644 --- a/morphlib/source_tests.py +++ b/morphlib/source_tests.py @@ -34,12 +34,11 @@ class SourceTests(unittest.TestCase): loader = morphlib.morphloader.MorphologyLoader() self.morphology = loader.load_from_string(self.morphology_text) self.filename = 'foo.morph' - self.source = morphlib.source.Source( - self.repo_name, self.original_ref, self.sha1, self.tree, - self.morphology, self.filename) - self.other = morphlib.source.Source( - self.repo_name, self.original_ref, self.sha1, self.tree, - self.morphology, self.filename) + self.source, = morphlib.source.make_sources(self.repo_name, + self.original_ref, + self.filename, + self.sha1, self.tree, + self.morphology) def test_sets_repo_name(self): self.assertEqual(self.source.repo_name, self.repo_name) diff --git a/morphlib/sourcepool.py b/morphlib/sourcepool.py index ec134c0a..6dfcb2c3 100644 --- a/morphlib/sourcepool.py +++ b/morphlib/sourcepool.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,12 +14,15 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import collections + + class SourcePool(object): '''Manage a collection of Source objects.''' def __init__(self): - self._sources = {} + self._sources = collections.defaultdict(dict) self._order = [] def _key(self, repo_name, original_ref, filename): @@ -30,8 +33,8 @@ class SourcePool(object): key = self._key(source.repo_name, source.original_ref, source.filename) - if key not in self._sources: - self._sources[key] = source + if key not in self._sources or source.name not in self._sources[key]: + self._sources[key][source.name] = source self._order.append(source) def lookup(self, repo_name, original_ref, filename): @@ -42,7 +45,7 @@ class SourcePool(object): ''' key = self._key(repo_name, original_ref, filename) - return self._sources[key] + return self._sources[key].values() def __iter__(self): '''Iterate over sources in the pool, in the order they were added.''' diff --git a/morphlib/sourcepool_tests.py b/morphlib/sourcepool_tests.py index 95264140..f3740049 100644 --- a/morphlib/sourcepool_tests.py +++ b/morphlib/sourcepool_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -22,6 +22,7 @@ import morphlib class DummySource(object): def __init__(self): + self.name = 'dummy' self.repo_name = 'repo' self.original_ref = 'original/ref' self.sha1 = 'dummy.sha1' @@ -50,14 +51,7 @@ class SourcePoolTests(unittest.TestCase): result = self.pool.lookup(self.source.repo_name, self.source.original_ref, self.source.filename) - self.assertEqual(result, self.source) - - def test_lookup_raises_keyerror_if_not_found(self): - self.assertRaises(KeyError, - self.pool.lookup, - self.source.repo_name, - self.source.original_ref, - self.source.filename) + self.assertEqual(result, [self.source]) def test_iterates_in_add_order(self): sources = [] diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index 0126b4d9..bfe0a716 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -65,8 +65,8 @@ class StagingArea(object): os.makedirs(dirname) def _dir_for_source(self, source, suffix): - basename = '%s.%s' % (str(source.morphology['name']), suffix) - dirname = os.path.join(self.dirname, basename) + dirname = os.path.join(self.dirname, + '%s.%s' % (str(source.name), suffix)) self._mkdir(dirname) return dirname diff --git a/morphlib/stagingarea_tests.py b/morphlib/stagingarea_tests.py index 52f495eb..dc43e4f6 100644 --- a/morphlib/stagingarea_tests.py +++ b/morphlib/stagingarea_tests.py @@ -37,6 +37,7 @@ class FakeSource(object): self.morphology = { 'name': 'le-name', } + self.name = 'le-name' class FakeApplication(object): |