Fix distbuild to work with per-source building

The serialisation code is now in sync with the code in morphlib.

FIXME: I didn't understand this:

 # We create separate sources for strata and systems,
 # this is a bit of a hack, but needed to allow
 # us to build strata and systems independently

so I commented it out for now.

2 files changed, 86 insertions, 86 deletions

diff --git a/distbuild/serialise.py b/distbuild/serialise.py
index d410b6cf..9b38c84f 100644
--- a/distbuild/serialise.py
+++ b/distbuild/serialise.py
@@ -38,8 +38,9 @@ def serialise_artifact(artifact):
             result['__%s' % x] = getattr(morphology, x)
         return result
     
-    def encode_source(source):
+    def encode_source(source, artifacts):
         source_dic = {
+            'name': source.name,
             'repo': None,
             'repo_name': source.repo_name,
             'original_ref': source.original_ref,
@@ -47,6 +48,11 @@ def serialise_artifact(artifact):
             'tree': source.tree,
             'morphology': encode_morphology(source.morphology),
             'filename': source.filename,
+            'cache_id': source.cache_id,
+            'cache_key': source.cache_key,
+
+            'dependencies': [str(id(artifacts[id(d)]))
+                for d in source.dependencies],
 
             # dict keys are converted to strings by json
             # so we encode the artifact ids as strings
@@ -59,63 +65,42 @@ def serialise_artifact(artifact):
             source_dic['prefix'] = source.prefix
         return source_dic
 
-    def encode_artifact(a, artifacts, source_id):
-        if artifact.source.morphology['kind'] == 'system':
-            arch = artifact.source.morphology['arch']
-        else:
-            arch = artifact.arch
-
+    def encode_artifact(a, source_id):
         return {
             'source_id': source_id,
             'name': a.name,
-            'cache_id': a.cache_id,
-            'cache_key': a.cache_key,
-            'dependencies': [str(id(artifacts[id(d)]))
-                for d in a.dependencies],
-            'arch': arch
         }
 
-    visited = set()
-    def traverse(a):
-        visited.add(a)
-        for dep in a.dependencies:
-            if dep in visited:
-                continue
-            for ret in traverse(dep):
-                yield ret
-        yield a
-    
-
     artifacts = {}
     encoded_artifacts = {}
     encoded_sources = {}
 
-    for a in traverse(artifact):
+    for a in artifact.walk():
         if id(a.source) not in encoded_sources:
-            if a.source.morphology['kind'] == 'chunk':
+            #if a.source.morphology['kind'] == 'chunk':
+            if True:
                 for (_, sa) in a.source.artifacts.iteritems():
                     if id(sa) not in artifacts:
                         artifacts[id(sa)] = sa
                         encoded_artifacts[id(sa)] = encode_artifact(sa,
-                            artifacts, id(a.source))
-            else:
+                            id(a.source))
+            #else:
                 # We create separate sources for strata and systems,
                 # this is a bit of a hack, but needed to allow
                 # us to build strata and systems independently
 
-                s = a.source
-                t = morphlib.source.Source(s.repo_name, s.original_ref,
-                    s.sha1, s.tree, s.morphology, s.filename)
+                #s = a.source
+                #t = morphlib.source.Source(s.repo_name, s.original_ref,
+                #    s.sha1, s.tree, s.morphology, s.filename)
 
-                t.artifacts = {a.name: a}
-                a.source = t
+                #t.artifacts = {a.name: a}
+                #a.source = t
 
-            encoded_sources[id(a.source)] = encode_source(a.source)
+            encoded_sources[id(a.source)] = encode_source(a.source, artifacts)
 
         if id(a) not in artifacts:
             artifacts[id(a)] = a
-            encoded_artifacts[id(a)] = encode_artifact(a, artifacts,
-                id(a.source))
+            encoded_artifacts[id(a)] = encode_artifact(a, id(a.source))
 
     encoded_artifacts['_root'] = str(id(artifact))
 
@@ -160,33 +145,45 @@ def deserialise_artifact(encoded):
         return morphology
 
     def decode_source(le_dict):
-        '''Convert a dict into a Source object.'''
+        '''Convert a dict into a Source object.
+
+        Do not set dependencies, that will be dealt with later.
+
+        '''
 
         morphology = decode_morphology(le_dict['morphology'])
-        source = morphlib.source.Source(le_dict['repo_name'],
-                                        le_dict['original_ref'],
-                                        le_dict['sha1'],
-                                        le_dict['tree'],
-                                        morphology,
-                                        le_dict['filename'])
+
+        sources = morphlib.source.make_sources(le_dict['repo_name'],
+                                               le_dict['original_ref'],
+                                               le_dict['filename'],
+                                               le_dict['sha1'],
+                                               le_dict['tree'],
+                                               morphology)
+
+        # The above function creates all sources produced by one morphology,
+        # but we're only deserialising one of them. Find it.
+        for source in sources:
+            if source.name == le_dict['name']:
+                break
+        else:
+            raise ValueError(
+                "Didn't find source %s in %s sources generated for %s." %
+                le_dict['name']. len(sources), le_dict['filename'])
+
+        source.cache_id = le_dict['cache_id']
+        source.cache_key = le_dict['cache_key']
 
         if morphology['kind'] == 'chunk':
             source.build_mode = le_dict['build_mode']
             source.prefix = le_dict['prefix']
+
         return source
-        
+
     def decode_artifact(artifact_dict, source):
-        '''Convert dict into an Artifact object.
-        
-        Do not set dependencies, that will be dealt with later.
-        
-        '''
+        '''Convert dict into an Artifact object.'''
 
         artifact = morphlib.artifact.Artifact(source, artifact_dict['name'])
-        artifact.cache_id = artifact_dict['cache_id']
-        artifact.cache_key = artifact_dict['cache_key']
-        artifact.arch = artifact_dict['arch']
-        artifact.source = source
+        #artifact.arch = artifact_dict['arch']
 
         return artifact
 
@@ -194,9 +191,6 @@ def deserialise_artifact(encoded):
     artifacts_dict = le_dicts['artifacts']
     sources_dict = le_dicts['sources']
 
-    artifact_ids = ([artifacts_dict['_root']] +
-        filter(lambda k: k != '_root', artifacts_dict.keys()))
-
     source_ids = [sid for sid in sources_dict.keys()]
 
     artifacts = {}
@@ -221,10 +215,7 @@ def deserialise_artifact(encoded):
             key = artifacts[artifact_id].name
             sources[source_id].artifacts[key] = artifacts[artifact_id]
 
-    # now add the dependencies
-    for artifact_id in artifact_ids:
-        artifact = artifacts[artifact_id]
-        artifact.dependencies = [artifacts[aid] for aid in
-            artifacts_dict[artifact_id]['dependencies']]
+        sources[source_id].dependencies = [artifacts[aid] for aid in
+                                           source_dict['dependencies']]
 
     return artifacts[artifacts_dict['_root']]
diff --git a/distbuild/serialise_tests.py b/distbuild/serialise_tests.py
index 2ad3a384..8e0a9413 100644
--- a/distbuild/serialise_tests.py
+++ b/distbuild/serialise_tests.py
@@ -19,14 +19,15 @@
 import unittest
 
 import distbuild
+import morphlib
 
 
 class MockMorphology(object):
 
-    def __init__(self, name):
+    def __init__(self, name, kind):
         self.dict = {
-            'name': '%s.morphology.name' % name,
-            'kind': '%s.morphology.kind' % name,
+            'name': name,
+            'kind': kind,
         }
         self.needs_staging_area = None
         self.needs_artifact_metadata_cached = None
@@ -40,21 +41,15 @@ class MockMorphology(object):
 
 class MockSource(object):
 
-    def __init__(self, name):
+    def __init__(self, name, kind):
+        self.name = name
         self.repo = None
         self.repo_name = '%s.source.repo_name' % name
         self.original_ref = '%s.source.original_ref' % name
         self.sha1 = '%s.source.sha1' % name
         self.tree = '%s.source.tree' % name
-        self.morphology = MockMorphology(name)
+        self.morphology = MockMorphology(name, kind)
         self.filename = '%s.source.filename' % name
-
-
-class MockArtifact(object):
-
-    def __init__(self, name):
-        self.source = MockSource(name)
-        self.name = name
         self.cache_id = {
             'blip': '%s.blip' % name,
             'integer': 42,
@@ -62,14 +57,26 @@ class MockArtifact(object):
         self.cache_key = '%s.cache_key' % name
         self.dependencies = []
 
+        if kind == 'chunk':
+            self.build_mode = '%s.source.build_mode' % name
+            self.prefix = '%s.source.prefix' % name
+
+
+
+def mock_artifact(name, kind):
+    source = MockSource(name, kind)
+    artifact = morphlib.artifact.Artifact(source, name)
+    source.artifacts = {name: artifact}
+    return artifact
+
 
 class SerialisationTests(unittest.TestCase):
 
     def setUp(self):
-        self.art1 = MockArtifact('name1')
-        self.art2 = MockArtifact('name2')
-        self.art3 = MockArtifact('name3')
-        self.art4 = MockArtifact('name4')
+        self.art1 = mock_artifact('name1', 'chunk')
+        self.art2 = mock_artifact('name2', 'chunk')
+        self.art3 = mock_artifact('name3', 'stratum')
+        self.art4 = mock_artifact('name4', 'system')
 
     def assertEqualMorphologies(self, a, b):
         self.assertEqual(sorted(a.keys()), sorted(b.keys()))
@@ -97,9 +104,11 @@ class SerialisationTests(unittest.TestCase):
         self.assertEqual(a.name, b.name)
         self.assertEqual(a.cache_id, b.cache_id)
         self.assertEqual(a.cache_key, b.cache_key)
-        self.assertEqual(len(a.dependencies), len(b.dependencies))
-        for i in range(len(a.dependencies)):
-            self.assertEqualArtifacts(a.dependencies[i], b.dependencies[i])
+        self.assertEqual(len(a.source.dependencies),
+                         len(b.source.dependencies))
+        for i in range(len(a.source.dependencies)):
+            self.assertEqualArtifacts(a.source.dependencies[i],
+                                      b.source.dependencies[i])
 
     def verify_round_trip(self, artifact):
         encoded = distbuild.serialise_artifact(artifact)
@@ -128,21 +137,21 @@ class SerialisationTests(unittest.TestCase):
         self.verify_round_trip(self.art1)
 
     def test_works_with_single_dependency(self):
-        self.art1.dependencies = [self.art2]
+        self.art1.source.dependencies = [self.art2]
         self.verify_round_trip(self.art1)
 
     def test_works_with_two_dependencies(self):
-        self.art1.dependencies = [self.art2, self.art3]
+        self.art1.source.dependencies = [self.art2, self.art3]
         self.verify_round_trip(self.art1)
 
     def test_works_with_two_levels_of_dependencies(self):
-        self.art2.dependencies = [self.art4]
-        self.art1.dependencies = [self.art2, self.art3]
+        self.art2.source.dependencies = [self.art4]
+        self.art1.source.dependencies = [self.art2, self.art3]
         self.verify_round_trip(self.art1)
 
     def test_works_with_dag(self):
-        self.art2.dependencies = [self.art4]
-        self.art3.dependencies = [self.art4]
-        self.art1.dependencies = [self.art2, self.art3]
+        self.art2.source.dependencies = [self.art4]
+        self.art3.source.dependencies = [self.art4]
+        self.art1.source.dependencies = [self.art2, self.art3]
         self.verify_round_trip(self.art1)