Fix and integrate distbuild unit tests

2 files changed, 113 insertions, 127 deletions

diff --git a/distbuild/serialise.py b/distbuild/serialise.py
index d410b6cf..0a60b0c2 100644
--- a/distbuild/serialise.py
+++ b/distbuild/serialise.py
@@ -22,11 +22,6 @@ import morphlib
 import logging
 
 
-morphology_attributes = [
-    'needs_artifact_metadata_cached',
-]
-
-
 def serialise_artifact(artifact):
     '''Serialise an Artifact object and its dependencies into string form.'''
 
@@ -34,24 +29,27 @@ def serialise_artifact(artifact):
         result = {}
         for key in morphology.keys():
             result[key] = morphology[key]
-        for x in morphology_attributes:
-            result['__%s' % x] = getattr(morphology, x)
         return result
     
     def encode_source(source):
         source_dic = {
+            'name': source.name,
             'repo': None,
             'repo_name': source.repo_name,
             'original_ref': source.original_ref,
             'sha1': source.sha1,
             'tree': source.tree,
-            'morphology': encode_morphology(source.morphology),
+            'morphology': str(id(source.morphology)),
             'filename': source.filename,
 
             # dict keys are converted to strings by json
             # so we encode the artifact ids as strings
             'artifact_ids': [str(id(artifact)) for (_, artifact)
                 in source.artifacts.iteritems()],
+            'cache_id': source.cache_id,
+            'cache_key': source.cache_key,
+            'dependencies': [str(id(d))
+                for d in source.dependencies],
         }
 
         if source.morphology['kind'] == 'chunk':
@@ -59,68 +57,42 @@ def serialise_artifact(artifact):
             source_dic['prefix'] = source.prefix
         return source_dic
 
-    def encode_artifact(a, artifacts, source_id):
-        if artifact.source.morphology['kind'] == 'system':
+    def encode_artifact(a):
+        if artifact.source.morphology['kind'] == 'system': # pragma: no cover
             arch = artifact.source.morphology['arch']
         else:
             arch = artifact.arch
 
         return {
-            'source_id': source_id,
+            'source_id': id(a.source),
             'name': a.name,
-            'cache_id': a.cache_id,
-            'cache_key': a.cache_key,
-            'dependencies': [str(id(artifacts[id(d)]))
-                for d in a.dependencies],
             'arch': arch
         }
 
-    visited = set()
-    def traverse(a):
-        visited.add(a)
-        for dep in a.dependencies:
-            if dep in visited:
-                continue
-            for ret in traverse(dep):
-                yield ret
-        yield a
-    
-
-    artifacts = {}
     encoded_artifacts = {}
     encoded_sources = {}
+    encoded_morphologies = {}
 
-    for a in traverse(artifact):
+    for a in artifact.walk():
         if id(a.source) not in encoded_sources:
-            if a.source.morphology['kind'] == 'chunk':
-                for (_, sa) in a.source.artifacts.iteritems():
-                    if id(sa) not in artifacts:
-                        artifacts[id(sa)] = sa
-                        encoded_artifacts[id(sa)] = encode_artifact(sa,
-                            artifacts, id(a.source))
-            else:
-                # We create separate sources for strata and systems,
-                # this is a bit of a hack, but needed to allow
-                # us to build strata and systems independently
-
-                s = a.source
-                t = morphlib.source.Source(s.repo_name, s.original_ref,
-                    s.sha1, s.tree, s.morphology, s.filename)
-
-                t.artifacts = {a.name: a}
-                a.source = t
-
+            for (_, sa) in a.source.artifacts.iteritems():
+                if id(sa) not in encoded_artifacts:
+                    encoded_artifacts[id(sa)] = encode_artifact(sa)
+            encoded_morphologies[id(a.source.morphology)] = encode_morphology(a.source.morphology)
             encoded_sources[id(a.source)] = encode_source(a.source)
 
-        if id(a) not in artifacts:
-            artifacts[id(a)] = a
-            encoded_artifacts[id(a)] = encode_artifact(a, artifacts,
-                id(a.source))
-
-    encoded_artifacts['_root'] = str(id(artifact))
+        if id(a) not in encoded_artifacts: # pragma: no cover
+            encoded_artifacts[id(a)] = encode_artifact(a)
 
     return json.dumps({'sources': encoded_sources,
-        'artifacts': encoded_artifacts})
+        'artifacts': encoded_artifacts,
+        'morphologies': encoded_morphologies,
+        'root_artifact': str(id(artifact)),
+        'default_split_rules': {
+            'chunk': morphlib.artifactsplitrule.DEFAULT_CHUNK_RULES,
+            'stratum': morphlib.artifactsplitrule.DEFAULT_STRATUM_RULES,
+        },
+    })
 
 
 def deserialise_artifact(encoded):
@@ -141,38 +113,25 @@ def deserialise_artifact(encoded):
         
         '''
         
-        class FakeMorphology(dict):
-        
-            def get_commands(self, which):
-                '''Get commands to run from a morphology or build system'''
-                if self[which] is None:
-                    attr = '_'.join(which.split('-'))
-                    bs = morphlib.buildsystem.lookup_build_system(
-                            self['build-system'])
-                    return getattr(bs, attr)
-                else:
-                    return self[which]
-
-        morphology = FakeMorphology(le_dict)
-        for x in morphology_attributes:
-            setattr(morphology, x, le_dict['__%s' % x])
-            del morphology['__%s' % x]
-        return morphology
-
-    def decode_source(le_dict):
+        return morphlib.morphology.Morphology(le_dict)
+
+    def decode_source(le_dict, morphology, split_rules):
         '''Convert a dict into a Source object.'''
 
-        morphology = decode_morphology(le_dict['morphology'])
-        source = morphlib.source.Source(le_dict['repo_name'],
+        source = morphlib.source.Source(le_dict['name'],
+                                        le_dict['repo_name'],
                                         le_dict['original_ref'],
                                         le_dict['sha1'],
                                         le_dict['tree'],
                                         morphology,
-                                        le_dict['filename'])
+                                        le_dict['filename'],
+                                        split_rules)
 
         if morphology['kind'] == 'chunk':
             source.build_mode = le_dict['build_mode']
             source.prefix = le_dict['prefix']
+        source.cache_id =  le_dict['cache_id']
+        source.cache_key = le_dict['cache_key']
         return source
         
     def decode_artifact(artifact_dict, source):
@@ -183,8 +142,6 @@ def deserialise_artifact(encoded):
         '''
 
         artifact = morphlib.artifact.Artifact(source, artifact_dict['name'])
-        artifact.cache_id = artifact_dict['cache_id']
-        artifact.cache_key = artifact_dict['cache_key']
         artifact.arch = artifact_dict['arch']
         artifact.source = source
 
@@ -193,18 +150,22 @@ def deserialise_artifact(encoded):
     le_dicts = json.loads(encoded)
     artifacts_dict = le_dicts['artifacts']
     sources_dict = le_dicts['sources']
+    morphologies_dict = le_dicts['morphologies']
+    root_artifact = le_dicts['root_artifact']
 
-    artifact_ids = ([artifacts_dict['_root']] +
-        filter(lambda k: k != '_root', artifacts_dict.keys()))
-
-    source_ids = [sid for sid in sources_dict.keys()]
+    artifact_ids = ([root_artifact] + artifacts_dict.keys())
 
     artifacts = {}
     sources = {}
+    morphologies = {id: decode_morphology(d)
+                    for (id, d) in morphologies_dict.iteritems()}
 
-    for source_id in source_ids:
-        source_dict = sources_dict[source_id]
-        sources[source_id] = decode_source(source_dict)
+    for source_id, source_dict in sources_dict.iteritems():
+        morphology = morphologies[source_dict['morphology']]
+        kind = morphology['kind']
+        ruler = getattr(morphlib.artifactsplitrule, 'unify_%s_matches' % kind)
+        rules = ruler(morphology, le_dicts['default_split_rules'][kind])
+        sources[source_id] = decode_source(source_dict, morphology, rules)
 
         # clear the source artifacts that get automatically generated
         # we want to add the ones that were sent to us
@@ -222,9 +183,9 @@ def deserialise_artifact(encoded):
             sources[source_id].artifacts[key] = artifacts[artifact_id]
 
     # now add the dependencies
-    for artifact_id in artifact_ids:
-        artifact = artifacts[artifact_id]
-        artifact.dependencies = [artifacts[aid] for aid in
-            artifacts_dict[artifact_id]['dependencies']]
+    for source_id, source_dict in sources_dict.iteritems():
+        source = sources[source_id]
+        source.dependencies = [artifacts[aid]
+                               for aid in source_dict['dependencies']]
 
-    return artifacts[artifacts_dict['_root']]
+    return artifacts[root_artifact]
diff --git a/distbuild/serialise_tests.py b/distbuild/serialise_tests.py
index 2ad3a384..70973346 100644
--- a/distbuild/serialise_tests.py
+++ b/distbuild/serialise_tests.py
@@ -23,13 +23,22 @@ import distbuild
 
 class MockMorphology(object):
 
-    def __init__(self, name):
+    def __init__(self, name, kind):
         self.dict = {
             'name': '%s.morphology.name' % name,
-            'kind': '%s.morphology.kind' % name,
+            'kind': kind,
+            'chunks': [],
+            'products': [
+                {
+                    'artifact': name,
+                    'include': [r'.*'],
+                },
+            ],
         }
-        self.needs_staging_area = None
-        self.needs_artifact_metadata_cached = None
+
+    @property
+    def needs_artifact_metadata_cached(self):
+        return self.dict['kind'] == 'stratum'
         
     def keys(self):
         return self.dict.keys()
@@ -40,36 +49,56 @@ class MockMorphology(object):
 
 class MockSource(object):
 
-    def __init__(self, name):
+    build_mode = 'staging'
+    prefix = '/usr'
+    def __init__(self, name, kind):
+        self.name = name
         self.repo = None
         self.repo_name = '%s.source.repo_name' % name
         self.original_ref = '%s.source.original_ref' % name
         self.sha1 = '%s.source.sha1' % name
         self.tree = '%s.source.tree' % name
-        self.morphology = MockMorphology(name)
+        self.morphology = MockMorphology(name, kind)
         self.filename = '%s.source.filename' % name
-
-
-class MockArtifact(object):
-
-    def __init__(self, name):
-        self.source = MockSource(name)
-        self.name = name
+        self.dependencies = []
         self.cache_id = {
             'blip': '%s.blip' % name,
             'integer': 42,
         }
         self.cache_key = '%s.cache_key' % name
-        self.dependencies = []
+        self.artifacts = {}
+
+
+class MockArtifact(object):
+
+    arch = 'testarch'
+
+    def __init__(self, name, kind):
+        self.source = MockSource(name, kind)
+        self.source.artifacts = {name: self}
+        self.name = name
+
+    def walk(self): # pragma: no cover
+        done = set()
+        
+        def depth_first(a):
+            if a not in done:
+                done.add(a)
+                for dep in a.source.dependencies:
+                    for ret in depth_first(dep):
+                        yield ret
+                yield a
+
+        return list(depth_first(self))
 
 
 class SerialisationTests(unittest.TestCase):
 
     def setUp(self):
-        self.art1 = MockArtifact('name1')
-        self.art2 = MockArtifact('name2')
-        self.art3 = MockArtifact('name3')
-        self.art4 = MockArtifact('name4')
+        self.art1 = MockArtifact('name1', 'stratum')
+        self.art2 = MockArtifact('name2', 'chunk')
+        self.art3 = MockArtifact('name3', 'chunk')
+        self.art4 = MockArtifact('name4', 'chunk')
 
     def assertEqualMorphologies(self, a, b):
         self.assertEqual(sorted(a.keys()), sorted(b.keys()))
@@ -77,11 +106,8 @@ class SerialisationTests(unittest.TestCase):
         a_values = [a[k] for k in keys]
         b_values = [b[k] for k in keys]
         self.assertEqual(a_values, b_values)
-        self.assertEqual(a.needs_staging_area, b.needs_staging_area)
         self.assertEqual(a.needs_artifact_metadata_cached, 
                          b.needs_artifact_metadata_cached)
-        self.assertEqual(a.needs_staging_area, 
-                         b.needs_staging_area)
 
     def assertEqualSources(self, a, b):
         self.assertEqual(a.repo, b.repo)
@@ -95,30 +121,29 @@ class SerialisationTests(unittest.TestCase):
     def assertEqualArtifacts(self, a, b):
         self.assertEqualSources(a.source, b.source)
         self.assertEqual(a.name, b.name)
-        self.assertEqual(a.cache_id, b.cache_id)
-        self.assertEqual(a.cache_key, b.cache_key)
-        self.assertEqual(len(a.dependencies), len(b.dependencies))
-        for i in range(len(a.dependencies)):
-            self.assertEqualArtifacts(a.dependencies[i], b.dependencies[i])
+        self.assertEqual(a.source.cache_id, b.source.cache_id)
+        self.assertEqual(a.source.cache_key, b.source.cache_key)
+        self.assertEqual(len(a.source.dependencies),
+                         len(b.source.dependencies))
+        for i in range(len(a.source.dependencies)):
+            self.assertEqualArtifacts(a.source.dependencies[i],
+                                      b.source.dependencies[i])
 
     def verify_round_trip(self, artifact):
         encoded = distbuild.serialise_artifact(artifact)
         decoded = distbuild.deserialise_artifact(encoded)
         self.assertEqualArtifacts(artifact, decoded)
         
-        def key(a):
-            return a.cache_key
-        
         objs = {}
         queue = [decoded]
         while queue:
             obj = queue.pop()
-            k = key(obj)
+            k = obj.source.cache_key
             if k in objs:
                 self.assertTrue(obj is objs[k])
             else:
                 objs[k] = obj
-            queue.extend(obj.dependencies)
+            queue.extend(obj.source.dependencies)
 
     def test_returns_string(self):
         encoded = distbuild.serialise_artifact(self.art1)
@@ -128,21 +153,21 @@ class SerialisationTests(unittest.TestCase):
         self.verify_round_trip(self.art1)
 
     def test_works_with_single_dependency(self):
-        self.art1.dependencies = [self.art2]
+        self.art1.source.dependencies = [self.art2]
         self.verify_round_trip(self.art1)
 
     def test_works_with_two_dependencies(self):
-        self.art1.dependencies = [self.art2, self.art3]
+        self.art1.source.dependencies = [self.art2, self.art3]
         self.verify_round_trip(self.art1)
 
     def test_works_with_two_levels_of_dependencies(self):
-        self.art2.dependencies = [self.art4]
-        self.art1.dependencies = [self.art2, self.art3]
+        self.art2.source.dependencies = [self.art4]
+        self.art1.source.dependencies = [self.art2, self.art3]
         self.verify_round_trip(self.art1)
 
     def test_works_with_dag(self):
-        self.art2.dependencies = [self.art4]
-        self.art3.dependencies = [self.art4]
-        self.art1.dependencies = [self.art2, self.art3]
+        self.art2.source.dependencies = [self.art4]
+        self.art3.source.dependencies = [self.art4]
+        self.art1.source.dependencies = [self.art2, self.art3]
         self.verify_round_trip(self.art1)