diff options
author | Jannis Pohlmann <jannis.pohlmann@codethink.co.uk> | 2012-01-31 16:38:18 +0000 |
---|---|---|
committer | Jannis Pohlmann <jannis.pohlmann@codethink.co.uk> | 2012-01-31 16:42:41 +0000 |
commit | c2104eb199916f9ef7e7b77d0eebf33619ed3ea9 (patch) | |
tree | 01eb904eda1461bf70ecf97aa8f6445d0bbfbb07 | |
parent | ab965be42d10f2608dc883ee0cc34a1cd0fe5058 (diff) | |
download | morph-c2104eb199916f9ef7e7b77d0eebf33619ed3ea9.tar.gz |
Properly unpack dependencies into the staging area in build-single.
This requires build-single to take a dependency context tuple when
building chunks of a stratum. This context tuple is the surrounding
stratum which is used to construct the dependency graph in the worker
and then do a breadth-first search to collect all dependencies that need
to be added to the staging area.
Implementing this required a few hash/eq changes in Blob, Morphology
and Treeish as well as a few adjustments in the corresponding unit
tests.
-rwxr-xr-x | morph | 45 | ||||
-rw-r--r-- | morphlib/blobs.py | 12 | ||||
-rw-r--r-- | morphlib/blobs_tests.py | 42 | ||||
-rw-r--r-- | morphlib/builder.py | 45 | ||||
-rw-r--r-- | morphlib/buildworker.py | 69 | ||||
-rw-r--r-- | morphlib/morphology.py | 7 | ||||
-rw-r--r-- | morphlib/morphology_tests.py | 32 |
7 files changed, 200 insertions, 52 deletions
@@ -228,20 +228,53 @@ class Morph(cliapp.Application): os.mkdir(self.settings['cachedir']) ret = [] - while len(args) >= 3: + if len(args) >= 3: repo, ref, filename = args[:3] args = args[3:] # derive a build order from the dependency graph graph = BuildDependencyGraph(source_manager, morph_loader, repo, ref, filename) - blob = graph.resolve() + first_blob = graph.resolve() blobs, order = graph.build_order() - self.msg('Building %s' % blob) - - # build things in this order - ret.append(builder.build_single(blob, blobs, order)) + # parse the second blob, if there is one + second_blob = None + if len(args) >= 3: + repo, ref, filename = args[:3] + args = args[3:] + + # load the blob manually + treeish = source_manager.get_treeish(repo, ref) + morphology = morph_loader.load(treeish, filename) + second_blob = morphlib.blobs.Blob.create_blob(morphology) + + try: + # find the corresponding blob object in the blobs + # returned from the dependency graph + second_blob = [x for x in blobs if x == second_blob][0] + except IndexError: + raise cliapp.AppException('%s and %s are unrelated' % + (first_blob, second_blob)) + + # build the single blob + if second_blob: + # verify that the two input blobs are valid + if first_blob.morph.kind != 'stratum': + raise cliapp.AppException('The first tuple %s needs to ' + 'refer to a stratum' % + first_blob) + if second_blob.morph.kind != 'chunk': + raise cliapp.AppException('The first tuple %s needs to ' + 'refer to a chunk' % second_blob) + + # build now + self.msg('Building %s' % second_blob) + ret.append(builder.build_single(second_blob, blobs, order)) + else: + # build the blob now + self.msg('Building %s' % first_blob) + ret.append(builder.build_single(first_blob, blobs, order)) # we may not have permission to tempdir.remove() ex = morphlib.execute.Execute('.', lambda msg: None) diff --git a/morphlib/blobs.py b/morphlib/blobs.py index 11f3e937..4c1190fb 100644 --- a/morphlib/blobs.py +++ b/morphlib/blobs.py @@ -35,10 +35,12 @@ class Blob(object): self.dependents = [] def add_parent(self, parent): - self.parents.append(parent) + if not parent in self.parents: + self.parents.append(parent) def remove_parent(self, parent): - self.parents.remove(parent) + if parent in self.parents: + self.parents.remove(parent) def add_dependency(self, other): self.dependencies.append(other) @@ -58,6 +60,12 @@ class Blob(object): else: return { self.morph.name: ['.'] } + def __eq__(self, other): + return self.morph == other.morph + + def __hash__(self): + return hash(self.morph) + def __str__(self): # pragma: no cover return str(self.morph) diff --git a/morphlib/blobs_tests.py b/morphlib/blobs_tests.py index f92658ee..b1062459 100644 --- a/morphlib/blobs_tests.py +++ b/morphlib/blobs_tests.py @@ -19,25 +19,27 @@ import unittest import morphlib +class FakeChunkMorph(object): + @property + def kind(self): + return 'chunk' + + +class FakeStratumMorph(object): + @property + def kind(self): + return 'stratum' + + class BlobsTests(unittest.TestCase): def test_create_a_chunk_blob(self): - class FakeChunkMorph(object): - @property - def kind(self): - return 'chunk' - morph = FakeChunkMorph() chunk = morphlib.blobs.Blob.create_blob(morph) self.assertTrue(isinstance(chunk, morphlib.blobs.Chunk)) self.assertEqual(morph, chunk.morph) def test_create_a_stratum_blob(self): - class FakeStratumMorph(object): - @property - def kind(self): - return 'stratum' - morph = FakeStratumMorph() stratum = morphlib.blobs.Blob.create_blob(morph) self.assertTrue(isinstance(stratum, morphlib.blobs.Stratum)) @@ -68,9 +70,9 @@ class BlobsTests(unittest.TestCase): self.assertRaises(TypeError, morphlib.blobs.Blob.create_blob, morph) def test_blob_with_parents(self): - blob1 = morphlib.blobs.Blob(None) - blob2 = morphlib.blobs.Blob(None) - blob3 = morphlib.blobs.Blob(None) + blob1 = morphlib.blobs.Blob(FakeChunkMorph()) + blob2 = morphlib.blobs.Blob(FakeStratumMorph()) + blob3 = morphlib.blobs.Blob(FakeStratumMorph()) self.assertEqual(len(blob1.parents), 0) @@ -105,7 +107,7 @@ class BlobsTests(unittest.TestCase): self.assertTrue(blob2 in blob1.dependencies) self.assertTrue(blob1 in blob2.dependents) - + self.assertTrue(blob1.depends_on(blob2)) blob2.add_dependency(blob1) @@ -138,6 +140,18 @@ class BlobsTests(unittest.TestCase): self.assertFalse(blob1.depends_on(blob2)) self.assertFalse(blob2.depends_on(blob1)) + def test_hashing_and_equality_checks(self): + morph = FakeChunkMorph() + blob1 = morphlib.blobs.Blob.create_blob(morph) + blob2 = morphlib.blobs.Blob.create_blob(morph) + blob3 = morphlib.blobs.Blob.create_blob(FakeChunkMorph()) + + self.assertEqual(hash(blob1), hash(blob2)) + self.assertEqual(blob1, blob2) + + self.assertNotEqual(hash(blob1), hash(blob3)) + self.assertNotEqual(blob1, blob3) + def test_chunks(self): settings = { 'git-base-url': [] } loader = morphlib.morphologyloader.MorphologyLoader(settings) diff --git a/morphlib/builder.py b/morphlib/builder.py index 53127223..9d7e066d 100644 --- a/morphlib/builder.py +++ b/morphlib/builder.py @@ -14,6 +14,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import collections import json import logging import os @@ -564,31 +565,33 @@ class Builder(object): # first pass: create builders for all blobs builders = {} for group in build_order: - for blob in group: - builder = self.create_blob_builder(blob) - builders[blob] = builder - - # second pass: walk the build order blob by blob, mark blobs - # to be staged for their parents, but do not build them - ret = [] - for group in build_order: - for blob in group: - built_items = builders[blob].builds() - for parent in blob.parents: - stage_items = [] - for name, filename in built_items.items(): - self.msg('Marking %s to be staged for %s' % - (name, parent)) - stage_items.append((name, filename)) - - parent_builder = builders[parent] - parent_builder.stage_items.extend(stage_items) + for b in group: + builder = self.create_blob_builder(b) + builders[b] = builder + + # second pass: mark all dependencies for staging + queue = collections.deque() + visited = set() + for dependency in blob.dependencies: + queue.append(dependency) + visited.add(dependency) + while len(queue) > 0: + dependency = queue.popleft() + built_items = builders[dependency].builds() + for name, filename in built_items.items(): + self.msg('Marking %s to be staged for %s' % (name, blob)) + builders[blob].stage_items.append((name, filename)) + for dep in dependency.dependencies: + if (dependency.morph.kind == 'stratum' and + not dep.morph.kind == 'chunk'): + if dep not in visited: + queue.append(dep) + visited.add(dep) # build the single blob now + ret = [] ret.append(builders[blob].build()) - self.indent_less() - return ret def create_blob_builder(self, blob): diff --git a/morphlib/buildworker.py b/morphlib/buildworker.py index 4304710d..74625a8c 100644 --- a/morphlib/buildworker.py +++ b/morphlib/buildworker.py @@ -141,7 +141,8 @@ class LocalBuildWorker(BuildWorker): def __init__(self, name, ident, app): BuildWorker.__init__(self, name, ident, app) - def run(self, repo, ref, filename, sudo, output, error): # pragma: no cover + def run(self, first_tuple, second_tuple, sudo, output, + error): # pragma: no cover ex = morphlib.execute.Execute('.', self.msg) # generate command line options @@ -149,7 +150,14 @@ class LocalBuildWorker(BuildWorker): cmdline = [] if sudo: cmdline.extend(['sudo']) - cmdline.extend(['morph', 'build-single', repo, ref, filename]) + cmdline.extend(['morph', 'build-single']) + cmdline.extend([first_tuple['repo'], + first_tuple['ref'], + first_tuple['filename']]) + if second_tuple: + cmdline.extend([second_tuple['repo'], + second_tuple['ref'], + second_tuple['filename']]) cmdline.extend(args) # run morph locally in a child process @@ -166,9 +174,23 @@ class LocalBuildWorker(BuildWorker): def build(self, blob): # pragma: no cover self.reset() self.blob = blob - args = (blob.morph.treeish.original_repo, - blob.morph.treeish.ref, - blob.morph.filename, + + first_tuple = None + if len(blob.parents) > 0: + first_tuple = { + 'repo': blob.parents[0].morph.treeish.original_repo, + 'ref': blob.parents[0].morph.treeish.ref, + 'filename': blob.parents[0].morph.filename, + } + + blob_tuple = { + 'repo': blob.morph.treeish.original_repo, + 'ref': blob.morph.treeish.ref, + 'filename': blob.morph.filename, + } + + args = (first_tuple if first_tuple else blob_tuple, + blob_tuple if first_tuple else None, blob.morph.kind == 'system', self._output, self._error) @@ -182,7 +204,8 @@ class RemoteBuildWorker(BuildWorker): BuildWorker.__init__(self, name, ident, app) self.hostname = ident - def run(self, repo, ref, filename, sudo, output, error): # pragma: no cover + def run(self, first_tuple, second_tuple, sudo, output, + error): # pragma: no cover ex = morphlib.execute.Execute('.', self.msg) # generate command line options @@ -193,11 +216,25 @@ class RemoteBuildWorker(BuildWorker): 'bash', '--login', '-c']) cmdline.extend(['"']) cmdline.extend(['morph', 'build-single', repo, ref, filename]) + cmdline.extend([first_tuple['repo'], + first_tuple['ref'], + first_tuple['filename']]) + if second_tuple: + cmdline.extend([second_tuple['repo'], + second_tuple['ref'], + second_tuple['filename']]) cmdline.extend(args) cmdline.extend(['"']) else: cmdline.extend(['fakeroot']) cmdline.extend(['morph', 'build-single', repo, ref, filename]) + cmdline.extend([first_tuple['repo'], + first_tuple['ref'], + first_tuple['filename']]) + if second_tuple: + cmdline.extend([second_tuple['repo'], + second_tuple['ref'], + second_tuple['filename']]) cmdline.extend(args) # run morph on the other machine @@ -214,9 +251,23 @@ class RemoteBuildWorker(BuildWorker): def build(self, blob): # pragma: no cover self.reset() self.blob = blob - args = (blob.morph.treeish.original_repo, - blob.morph.treeish.ref, - blob.morph.filename, + + first_tuple = None + if len(blob.parents) > 0: + first_tuple = { + 'repo': blob.parents[0].morph.treeish.original_repo, + 'ref': blob.parents[0].morph.treeish.ref, + 'filename': blob.parents[0].morph.filename, + } + + blob_tuple = { + 'repo': blob.morph.treeish.original_repo, + 'ref': blob.morph.treeish.ref, + 'filename': blob.morph.filename, + } + + args = (first_tuple if first_tuple else blob_tuple, + blob_tuple if first_tuple else None, blob.morph.kind == 'system', self._output, self._error) diff --git a/morphlib/morphology.py b/morphlib/morphology.py index 2b8c0551..734c4e3d 100644 --- a/morphlib/morphology.py +++ b/morphlib/morphology.py @@ -108,6 +108,13 @@ class Morphology(object): def test_stories(self): return self._dict.get('test-stories', []) + def __eq__(self, other): + return (self.filename == other.filename and + self.treeish == other.treeish) + + def __hash__(self): + return hash((self.filename, self.treeish)) + def __str__(self): # pragma: no cover return '%s|%s|%s' % (self.treeish.original_repo, self.treeish.ref, diff --git a/morphlib/morphology_tests.py b/morphlib/morphology_tests.py index 54cdbb18..575aaeec 100644 --- a/morphlib/morphology_tests.py +++ b/morphlib/morphology_tests.py @@ -172,3 +172,35 @@ class MorphologyTests(unittest.TestCase): self.assertEqual(morph.disk_size, '1G') self.assertEqual(morph.strata, ['foo', 'bar']) self.assertEqual(morph.test_stories, ['test-1', 'test-2']) + + def test_hashing_and_equality_checks(self): + mockfile1 = MockFile(''' + { + "name": "foo", + "kind": "chunk" + }''') + mockfile1.name = 'mockfile1' + mockfile2 = MockFile(''' + { + "name": "foo", + "kind": "chunk" + }''') + mockfile2.name = 'mockfile1' + mockfile3 = MockFile(''' + { + "name": "bar", + "kind": "chunk" + }''') + mockfile3.name = 'mockfile2' + + treeish = FakeTreeish() + + morph1 = morphlib.morphology.Morphology(treeish, mockfile1) + morph2 = morphlib.morphology.Morphology(treeish, mockfile2) + morph3 = morphlib.morphology.Morphology(treeish, mockfile3) + + self.assertEqual(hash(morph1), hash(morph2)) + self.assertEqual(morph1, morph2) + + self.assertNotEqual(hash(morph1), hash(morph3)) + self.assertNotEqual(morph1, morph3) |