Properly unpack dependencies into the staging area in build-single.

This requires build-single to take a dependency context tuple when
building chunks of a stratum. This context tuple is the surrounding
stratum which is used to construct the dependency graph in the worker
and then do a breadth-first search to collect all dependencies that need
to be added to the staging area.

Implementing this required a few hash/eq changes in Blob, Morphology
and Treeish as well as a few adjustments in the corresponding unit
tests.

7 files changed, 200 insertions, 52 deletions

diff --git a/morph b/morph
index ce2838ac..db86bb69 100755
--- a/morph
+++ b/morph
@@ -228,20 +228,53 @@ class Morph(cliapp.Application):
             os.mkdir(self.settings['cachedir'])
 
         ret = []
-        while len(args) >= 3:
+        if len(args) >= 3:
             repo, ref, filename = args[:3]
             args = args[3:]
 
             # derive a build order from the dependency graph
             graph = BuildDependencyGraph(source_manager, morph_loader,
                                          repo, ref, filename)
-            blob = graph.resolve()
+            first_blob = graph.resolve()
             blobs, order = graph.build_order()
 
-            self.msg('Building %s' % blob)
-
-            # build things in this order
-            ret.append(builder.build_single(blob, blobs, order))
+            # parse the second blob, if there is one
+            second_blob = None
+            if len(args) >= 3:
+                repo, ref, filename = args[:3]
+                args = args[3:]
+
+                # load the blob manually
+                treeish = source_manager.get_treeish(repo, ref)
+                morphology = morph_loader.load(treeish, filename)
+                second_blob = morphlib.blobs.Blob.create_blob(morphology)
+
+                try:
+                    # find the corresponding blob object in the blobs
+                    # returned from the dependency graph
+                    second_blob = [x for x in blobs if x == second_blob][0]
+                except IndexError:
+                    raise cliapp.AppException('%s and %s are unrelated' %
+                                              (first_blob, second_blob))
+
+            # build the single blob
+            if second_blob:
+                # verify that the two input blobs are valid
+                if first_blob.morph.kind != 'stratum':
+                    raise cliapp.AppException('The first tuple %s needs to '
+                                              'refer to a stratum' %
+                                              first_blob)
+                if second_blob.morph.kind != 'chunk':
+                    raise cliapp.AppException('The first tuple %s needs to '
+                                              'refer to a chunk' % second_blob)
+
+                # build now
+                self.msg('Building %s' % second_blob)
+                ret.append(builder.build_single(second_blob, blobs, order))
+            else:
+                # build the blob now
+                self.msg('Building %s' % first_blob)
+                ret.append(builder.build_single(first_blob, blobs, order))
 
         # we may not have permission to tempdir.remove()
         ex = morphlib.execute.Execute('.', lambda msg: None)
diff --git a/morphlib/blobs.py b/morphlib/blobs.py
index 11f3e937..4c1190fb 100644
--- a/morphlib/blobs.py
+++ b/morphlib/blobs.py
@@ -35,10 +35,12 @@ class Blob(object):
         self.dependents = []
 
     def add_parent(self, parent):
-        self.parents.append(parent)
+        if not parent in self.parents:
+            self.parents.append(parent)
 
     def remove_parent(self, parent):
-        self.parents.remove(parent)
+        if parent in self.parents:
+            self.parents.remove(parent)
 
     def add_dependency(self, other):
         self.dependencies.append(other)
@@ -58,6 +60,12 @@ class Blob(object):
         else:
             return { self.morph.name: ['.'] }
 
+    def __eq__(self, other):
+        return self.morph == other.morph
+
+    def __hash__(self):
+        return hash(self.morph)
+
     def __str__(self): # pragma: no cover
         return str(self.morph)
 
diff --git a/morphlib/blobs_tests.py b/morphlib/blobs_tests.py
index f92658ee..b1062459 100644
--- a/morphlib/blobs_tests.py
+++ b/morphlib/blobs_tests.py
@@ -19,25 +19,27 @@ import unittest
 import morphlib
 
 
+class FakeChunkMorph(object):
+    @property
+    def kind(self):
+        return 'chunk'
+
+
+class FakeStratumMorph(object):
+    @property
+    def kind(self):
+        return 'stratum'
+
+
 class BlobsTests(unittest.TestCase):
 
     def test_create_a_chunk_blob(self):
-        class FakeChunkMorph(object):
-            @property
-            def kind(self):
-                return 'chunk'
-
         morph = FakeChunkMorph()
         chunk = morphlib.blobs.Blob.create_blob(morph)
         self.assertTrue(isinstance(chunk, morphlib.blobs.Chunk))
         self.assertEqual(morph, chunk.morph)
 
     def test_create_a_stratum_blob(self):
-        class FakeStratumMorph(object):
-            @property
-            def kind(self):
-                return 'stratum'
-
         morph = FakeStratumMorph()
         stratum = morphlib.blobs.Blob.create_blob(morph)
         self.assertTrue(isinstance(stratum, morphlib.blobs.Stratum))
@@ -68,9 +70,9 @@ class BlobsTests(unittest.TestCase):
         self.assertRaises(TypeError, morphlib.blobs.Blob.create_blob, morph)
 
     def test_blob_with_parents(self):
-        blob1 = morphlib.blobs.Blob(None)
-        blob2 = morphlib.blobs.Blob(None)
-        blob3 = morphlib.blobs.Blob(None)
+        blob1 = morphlib.blobs.Blob(FakeChunkMorph())
+        blob2 = morphlib.blobs.Blob(FakeStratumMorph())
+        blob3 = morphlib.blobs.Blob(FakeStratumMorph())
 
         self.assertEqual(len(blob1.parents), 0)
 
@@ -105,7 +107,7 @@ class BlobsTests(unittest.TestCase):
 
         self.assertTrue(blob2 in blob1.dependencies)
         self.assertTrue(blob1 in blob2.dependents)
-        
+
         self.assertTrue(blob1.depends_on(blob2))
 
         blob2.add_dependency(blob1)
@@ -138,6 +140,18 @@ class BlobsTests(unittest.TestCase):
         self.assertFalse(blob1.depends_on(blob2))
         self.assertFalse(blob2.depends_on(blob1))
 
+    def test_hashing_and_equality_checks(self):
+        morph = FakeChunkMorph()
+        blob1 = morphlib.blobs.Blob.create_blob(morph)
+        blob2 = morphlib.blobs.Blob.create_blob(morph)
+        blob3 = morphlib.blobs.Blob.create_blob(FakeChunkMorph())
+
+        self.assertEqual(hash(blob1), hash(blob2))
+        self.assertEqual(blob1, blob2)
+
+        self.assertNotEqual(hash(blob1), hash(blob3))
+        self.assertNotEqual(blob1, blob3)
+
     def test_chunks(self):
         settings = { 'git-base-url': [] }
         loader = morphlib.morphologyloader.MorphologyLoader(settings)
diff --git a/morphlib/builder.py b/morphlib/builder.py
index 53127223..9d7e066d 100644
--- a/morphlib/builder.py
+++ b/morphlib/builder.py
@@ -14,6 +14,7 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 
+import collections
 import json
 import logging
 import os
@@ -564,31 +565,33 @@ class Builder(object):
         # first pass: create builders for all blobs
         builders = {}
         for group in build_order:
-            for blob in group:
-                builder = self.create_blob_builder(blob)
-                builders[blob] = builder
-
-        # second pass: walk the build order blob by blob, mark blobs
-        # to be staged for their parents, but do not build them
-        ret = []
-        for group in build_order:
-            for blob in group:
-                built_items = builders[blob].builds()
-                for parent in blob.parents:
-                    stage_items = []
-                    for name, filename in built_items.items():
-                        self.msg('Marking %s to be staged for %s' %
-                                 (name, parent))
-                        stage_items.append((name, filename))
-
-                    parent_builder = builders[parent]
-                    parent_builder.stage_items.extend(stage_items)
+            for b in group:
+                builder = self.create_blob_builder(b)
+                builders[b] = builder
+
+        # second pass: mark all dependencies for staging
+        queue = collections.deque()
+        visited = set()
+        for dependency in blob.dependencies:
+            queue.append(dependency)
+            visited.add(dependency)
+        while len(queue) > 0:
+            dependency = queue.popleft()
+            built_items = builders[dependency].builds()
+            for name, filename in built_items.items():
+                self.msg('Marking %s to be staged for %s' % (name, blob))
+                builders[blob].stage_items.append((name, filename))
+            for dep in dependency.dependencies:
+                if (dependency.morph.kind == 'stratum' and
+                        not dep.morph.kind == 'chunk'):
+                    if dep not in visited:
+                        queue.append(dep)
+                        visited.add(dep)
 
         # build the single blob now
+        ret = []
         ret.append(builders[blob].build())
-
         self.indent_less()
-
         return ret
 
     def create_blob_builder(self, blob):
diff --git a/morphlib/buildworker.py b/morphlib/buildworker.py
index 4304710d..74625a8c 100644
--- a/morphlib/buildworker.py
+++ b/morphlib/buildworker.py
@@ -141,7 +141,8 @@ class LocalBuildWorker(BuildWorker):
     def __init__(self, name, ident, app):
         BuildWorker.__init__(self, name, ident, app)
 
-    def run(self, repo, ref, filename, sudo, output, error): # pragma: no cover
+    def run(self, first_tuple, second_tuple, sudo, output,
+            error): # pragma: no cover
         ex = morphlib.execute.Execute('.', self.msg)
 
         # generate command line options
@@ -149,7 +150,14 @@ class LocalBuildWorker(BuildWorker):
         cmdline = []
         if sudo:
             cmdline.extend(['sudo'])
-        cmdline.extend(['morph', 'build-single', repo, ref, filename])
+        cmdline.extend(['morph', 'build-single'])
+        cmdline.extend([first_tuple['repo'],
+                        first_tuple['ref'],
+                        first_tuple['filename']])
+        if second_tuple:
+            cmdline.extend([second_tuple['repo'],
+                            second_tuple['ref'],
+                            second_tuple['filename']])
         cmdline.extend(args)
 
         # run morph locally in a child process
@@ -166,9 +174,23 @@ class LocalBuildWorker(BuildWorker):
     def build(self, blob): # pragma: no cover
         self.reset()
         self.blob = blob
-        args = (blob.morph.treeish.original_repo,
-                blob.morph.treeish.ref,
-                blob.morph.filename,
+
+        first_tuple = None
+        if len(blob.parents) > 0:
+            first_tuple = {
+                'repo': blob.parents[0].morph.treeish.original_repo,
+                'ref': blob.parents[0].morph.treeish.ref,
+                'filename': blob.parents[0].morph.filename,
+            }
+
+        blob_tuple = {
+            'repo': blob.morph.treeish.original_repo,
+            'ref': blob.morph.treeish.ref,
+            'filename': blob.morph.filename,
+        }
+
+        args = (first_tuple if first_tuple else blob_tuple,
+                blob_tuple if first_tuple else None,
                 blob.morph.kind == 'system',
                 self._output,
                 self._error)
@@ -182,7 +204,8 @@ class RemoteBuildWorker(BuildWorker):
         BuildWorker.__init__(self, name, ident, app)
         self.hostname = ident
 
-    def run(self, repo, ref, filename, sudo, output, error): # pragma: no cover
+    def run(self, first_tuple, second_tuple, sudo, output,
+            error): # pragma: no cover
         ex = morphlib.execute.Execute('.', self.msg)
 
         # generate command line options
@@ -193,11 +216,25 @@ class RemoteBuildWorker(BuildWorker):
                             'bash', '--login', '-c'])
             cmdline.extend(['"'])
             cmdline.extend(['morph', 'build-single', repo, ref, filename])
+            cmdline.extend([first_tuple['repo'],
+                            first_tuple['ref'],
+                            first_tuple['filename']])
+            if second_tuple:
+                cmdline.extend([second_tuple['repo'],
+                                second_tuple['ref'],
+                                second_tuple['filename']])
             cmdline.extend(args)
             cmdline.extend(['"'])
         else:
             cmdline.extend(['fakeroot'])
             cmdline.extend(['morph', 'build-single', repo, ref, filename])
+            cmdline.extend([first_tuple['repo'],
+                            first_tuple['ref'],
+                            first_tuple['filename']])
+            if second_tuple:
+                cmdline.extend([second_tuple['repo'],
+                                second_tuple['ref'],
+                                second_tuple['filename']])
             cmdline.extend(args)
 
         # run morph on the other machine
@@ -214,9 +251,23 @@ class RemoteBuildWorker(BuildWorker):
     def build(self, blob): # pragma: no cover
         self.reset()
         self.blob = blob
-        args = (blob.morph.treeish.original_repo,
-                blob.morph.treeish.ref,
-                blob.morph.filename,
+
+        first_tuple = None
+        if len(blob.parents) > 0:
+            first_tuple = {
+                'repo': blob.parents[0].morph.treeish.original_repo,
+                'ref': blob.parents[0].morph.treeish.ref,
+                'filename': blob.parents[0].morph.filename,
+            }
+
+        blob_tuple = {
+            'repo': blob.morph.treeish.original_repo,
+            'ref': blob.morph.treeish.ref,
+            'filename': blob.morph.filename,
+        }
+
+        args = (first_tuple if first_tuple else blob_tuple,
+                blob_tuple if first_tuple else None,
                 blob.morph.kind == 'system',
                 self._output,
                 self._error)
diff --git a/morphlib/morphology.py b/morphlib/morphology.py
index 2b8c0551..734c4e3d 100644
--- a/morphlib/morphology.py
+++ b/morphlib/morphology.py
@@ -108,6 +108,13 @@ class Morphology(object):
     def test_stories(self):
         return self._dict.get('test-stories', [])
 
+    def __eq__(self, other):
+        return (self.filename == other.filename and
+                self.treeish == other.treeish)
+
+    def __hash__(self):
+        return hash((self.filename, self.treeish))
+
     def __str__(self): # pragma: no cover
         return '%s|%s|%s' % (self.treeish.original_repo,
                              self.treeish.ref,
diff --git a/morphlib/morphology_tests.py b/morphlib/morphology_tests.py
index 54cdbb18..575aaeec 100644
--- a/morphlib/morphology_tests.py
+++ b/morphlib/morphology_tests.py
@@ -172,3 +172,35 @@ class MorphologyTests(unittest.TestCase):
         self.assertEqual(morph.disk_size, '1G')
         self.assertEqual(morph.strata, ['foo', 'bar'])
         self.assertEqual(morph.test_stories, ['test-1', 'test-2'])
+
+    def test_hashing_and_equality_checks(self):
+        mockfile1 = MockFile('''
+                        {
+                            "name": "foo",
+                            "kind": "chunk"
+                        }''')
+        mockfile1.name = 'mockfile1'
+        mockfile2 = MockFile('''
+                        {
+                            "name": "foo",
+                            "kind": "chunk"
+                        }''')
+        mockfile2.name = 'mockfile1'
+        mockfile3 = MockFile('''
+                        {
+                            "name": "bar",
+                            "kind": "chunk"
+                        }''')
+        mockfile3.name = 'mockfile2'
+
+        treeish = FakeTreeish()
+
+        morph1 = morphlib.morphology.Morphology(treeish, mockfile1)
+        morph2 = morphlib.morphology.Morphology(treeish, mockfile2)
+        morph3 = morphlib.morphology.Morphology(treeish, mockfile3)
+
+        self.assertEqual(hash(morph1), hash(morph2))
+        self.assertEqual(morph1, morph2)
+
+        self.assertNotEqual(hash(morph1), hash(morph3))
+        self.assertNotEqual(morph1, morph3)