Create multiple sources per stratum morphology

Building per-artifact results in undesirable behaviour,
as multiple artifacts are produced for every chunk build.
It therefore makes more sense to build per-source.

This implies that actually, the model of one source per
morphology is wrong and we should move the dependencies
into the source.

Unlike chunks however, where every chunk artifact has the
same dependencies, stratum artifacts can have different
dependencies.

So before we can move the dependencies into the Source,
we need to have as many Sources as Stratum Artifacts.

6 files changed, 112 insertions, 66 deletions

diff --git a/morphlib/app.py b/morphlib/app.py
index 88eb58a4..fae4c84b 100644
--- a/morphlib/app.py
+++ b/morphlib/app.py
@@ -290,9 +290,11 @@ class Morph(cliapp.Application):
         pool = morphlib.sourcepool.SourcePool()
 
         def add_to_pool(reponame, ref, filename, absref, tree, morphology):
-            source = morphlib.source.Source(reponame, ref, absref, tree,
-                                            morphology, filename)
-            pool.add(source)
+            sources = morphlib.source.make_sources(reponame, ref,
+                                                   filename, absref,
+                                                   tree, morphology)
+            for source in sources:
+                pool.add(source)
 
         self.traverse_morphs(repo, ref, [filename], lrc, rrc,
                              update=not self.settings['no-git-update'],
diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py
index 49e03664..a60a8989 100644
--- a/morphlib/artifactresolver.py
+++ b/morphlib/artifactresolver.py
@@ -68,7 +68,7 @@ class ArtifactResolver(object):
         while queue:
             source = queue.popleft()
 
-            if source.morphology['kind'] == 'system':
+            if source.morphology['kind'] == 'system': # pragma: no cover
                 systems = [source.artifacts[name]
                            for name in source.split_rules.artifacts]
 
@@ -85,8 +85,11 @@ class ArtifactResolver(object):
                         artifacts.append(artifact)
                         self._added_artifacts.add(artifact)
             elif source.morphology['kind'] == 'stratum':
+                # Iterate split_rules.artifacts, rather than
+                # artifacts.values() to preserve ordering
                 strata = [source.artifacts[name]
-                          for name in source.split_rules.artifacts]
+                          for name in source.split_rules.artifacts
+                          if name in source.artifacts]
 
                 # If we were not given systems, return the strata here,
                 # rather than have the systems return them.
@@ -126,26 +129,30 @@ class ArtifactResolver(object):
                        if x.morphology['kind'] != 'chunk']
             return collections.deque(sources)
 
-    def _resolve_system_dependencies(self, systems, source, queue):
+    def _resolve_system_dependencies(self, systems,
+                                     source, queue): # pragma: no cover
         artifacts = []
 
         for info in source.morphology['strata']:
-            stratum_source = self._source_pool.lookup(
+            for stratum_source in self._source_pool.lookup(
                 info.get('repo') or source.repo_name,
                 info.get('ref') or source.original_ref,
-                morphlib.util.sanitise_morphology_path(info['morph']))
-            stratum_name = stratum_source.morphology['name']
+                morphlib.util.sanitise_morphology_path(info['morph'])):
 
-            matches, overlaps, unmatched = source.split_rules.partition(
-                    ((stratum_name, sta_name) for sta_name
-                     in stratum_source.split_rules.artifacts))
-            for system in systems:
-                for (stratum_name, sta_name) in matches[system.name]:
-                    stratum = stratum_source.artifacts[sta_name]
-                    system.add_dependency(stratum)
-                    artifacts.append(stratum)
+                stratum_morph_name = stratum_source.morphology['name']
+
+                matches, overlaps, unmatched = source.split_rules.partition(
+                        ((stratum_morph_name, sta_name) for sta_name
+                         in stratum_source.split_rules.artifacts))
+                for system in systems:
+                    for (stratum_name, sta_name) in matches[system.name]:
+                        if sta_name in stratum_source.artifacts:
+                            stratum_artifact = \
+                                stratum_source.artifacts[sta_name]
+                            system.add_dependency(stratum_artifact)
+                            artifacts.append(stratum_artifact)
 
-            queue.append(stratum_source)
+                queue.append(stratum_source)
 
         return artifacts
 
@@ -155,28 +162,32 @@ class ArtifactResolver(object):
         stratum_build_depends = []
 
         for stratum_info in source.morphology.get('build-depends') or []:
-            other_source = self._source_pool.lookup(
+            for other_source in self._source_pool.lookup(
                 stratum_info.get('repo') or source.repo_name,
                 stratum_info.get('ref') or source.original_ref,
-                morphlib.util.sanitise_morphology_path(stratum_info['morph']))
+                morphlib.util.sanitise_morphology_path(stratum_info['morph'])):
 
-            # Make every stratum artifact this stratum source produces
-            # depend on every stratum artifact the other stratum source
-            # produces.
-            for sta_name in other_source.split_rules.artifacts:
-                other_stratum = other_source.artifacts[sta_name]
+                # Make every stratum artifact this stratum source produces
+                # depend on every stratum artifact the other stratum source
+                # produces.
+                for sta_name in other_source.split_rules.artifacts:
+                    # Strata have split rules for artifacts they don't build,
+                    # since they need to know to yield a match to its sibling
+                    if sta_name not in other_source.artifacts:
+                        continue
+                    other_stratum = other_source.artifacts[sta_name]
 
-                stratum_build_depends.append(other_stratum)
+                    stratum_build_depends.append(other_stratum)
 
-                artifacts.append(other_stratum)
+                    artifacts.append(other_stratum)
 
-                for stratum in strata:
-                    if other_stratum.depends_on(stratum):
-                        raise MutualDependencyError(stratum, other_stratum)
+                    for stratum in strata:
+                        if other_stratum.depends_on(stratum):
+                            raise MutualDependencyError(stratum, other_stratum)
 
-                    stratum.add_dependency(other_stratum)
+                        stratum.add_dependency(other_stratum)
 
-            queue.append(other_source)
+                queue.append(other_source)
 
         # 'name' here is the chunk artifact name
         name_to_processed_artifacts = {}
@@ -187,9 +198,9 @@ class ArtifactResolver(object):
             chunk_source = self._source_pool.lookup(
                 info['repo'],
                 info['ref'],
-                filename)
+                filename)[0]
 
-            chunk_name = chunk_source.morphology['name']
+            chunk_name = chunk_source.name
 
             # Resolve now to avoid a search for the parent morphology later
             chunk_source.build_mode = info['build-mode']
@@ -205,7 +216,7 @@ class ArtifactResolver(object):
                     chunk_artifact.add_dependency(other_stratum)
 
             # Add dependencies between chunks mentioned in this stratum
-            for name in build_depends:
+            for name in build_depends: # pragma: no cover
                 if name not in name_to_processed_artifacts:
                     raise DependencyOrderError(
                         source, info['name'], name)
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index 352b43d2..436e23eb 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -208,7 +208,7 @@ class BuildCommand(object):
             # a build-dependency, then they must both have the same Repository
             # and Ref specified.
             if src.morphology['kind'] == 'stratum':
-                name = src.morphology['name']
+                name = src.name
                 ref = src.sha1[:7]
                 self.app.status(msg='Stratum [%(name)s] version is %(ref)s', 
                                 name=name, ref=ref)
@@ -236,18 +236,18 @@ class BuildCommand(object):
             logging.debug(
                 'Validating cross ref to %s:%s:%s' %
                     (repo_name, ref, filename))
-            other = srcpool.lookup(repo_name, ref, filename)
-            if other.morphology['kind'] != wanted:
-                raise morphlib.Error(
-                    '%s %s references %s:%s:%s which is a %s, '
-                        'instead of a %s' %
-                        (src.morphology['kind'],
-                         src.morphology['name'],
-                         repo_name,
-                         ref,
-                         filename,
-                         other.morphology['kind'],
-                         wanted))
+            for other in srcpool.lookup(repo_name, ref, filename):
+                if other.morphology['kind'] != wanted:
+                    raise morphlib.Error(
+                        '%s %s references %s:%s:%s which is a %s, '
+                            'instead of a %s' %
+                            (src.morphology['kind'],
+                             src.name,
+                             repo_name,
+                             ref,
+                             filename,
+                             other.morphology['kind'],
+                             wanted))
 
     def _find_root_artifacts(self, artifacts):
         '''Find all the root artifacts among a set of artifacts in a DAG.
diff --git a/morphlib/source.py b/morphlib/source.py
index d0f69a28..3d7e5a0f 100644
--- a/morphlib/source.py
+++ b/morphlib/source.py
@@ -35,8 +35,9 @@ class Source(object):
 
     '''
 
-    def __init__(self, repo_name, original_ref, sha1, tree, morphology,
-            filename):
+    def __init__(self, name, repo_name, original_ref, sha1, tree, morphology,
+            filename, split_rules):
+        self.name = name
         self.repo = None
         self.repo_name = repo_name
         self.original_ref = original_ref
@@ -45,17 +46,46 @@ class Source(object):
         self.morphology = morphology
         self.filename = filename
 
-        kind = morphology['kind']
-        unifier = getattr(morphlib.artifactsplitrule,
-                          'unify_%s_matches' % kind)
-        self.split_rules = unifier(morphology)
-        self.artifacts = {name: morphlib.artifact.Artifact(self, name)
-                          for name in self.split_rules.artifacts}
+        self.split_rules = split_rules
+        self.artifacts = None
 
     def __str__(self):  # pragma: no cover
-        return '%s|%s|%s' % (self.repo_name,
-                             self.original_ref,
-                             self.filename)
+        return '%s|%s|%s|%s' % (self.repo_name,
+                                self.original_ref,
+                                self.filename,
+                                self.name)
 
     def __repr__(self): # pragma: no cover
         return 'Source(%s)' % str(self)
+
+
+def make_sources(reponame, ref, filename, absref, tree, morphology):
+    kind = morphology['kind']
+    if kind in ('system', 'chunk'):
+        unifier = getattr(morphlib.artifactsplitrule,
+                          'unify_%s_matches' % kind)
+        split_rules = unifier(morphology)
+        # chunk and system sources are named after the morphology
+        source_name = morphology['name']
+        source = morphlib.source.Source(source_name, reponame, ref,
+                                        absref, tree, morphology,
+                                        filename, split_rules)
+        source.artifacts = {name: morphlib.artifact.Artifact(source, name)
+                     for name in split_rules.artifacts}
+        yield source
+    elif kind == 'stratum': # pragma: no cover
+        unifier = morphlib.artifactsplitrule.unify_stratum_matches
+        split_rules = unifier(morphology)
+        for name in split_rules.artifacts:
+            source = morphlib.source.Source(
+                name, # stratum source name is artifact name
+                reponame, ref, absref, tree, morphology, filename,
+                # stratum sources need to match the unified
+                # split rules, so they know to yield the match
+                # to a different source
+                split_rules)
+            source.artifacts = {name: morphlib.artifact.Artifact(source, name)}
+            yield source
+    else:
+        # cluster morphologies don't have sources
+        pass
diff --git a/morphlib/sourcepool.py b/morphlib/sourcepool.py
index ec134c0a..6dfcb2c3 100644
--- a/morphlib/sourcepool.py
+++ b/morphlib/sourcepool.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012  Codethink Limited
+# Copyright (C) 2012-2014  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -14,12 +14,15 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 
+import collections
+
+
 class SourcePool(object):
 
     '''Manage a collection of Source objects.'''
 
     def __init__(self):
-        self._sources = {}
+        self._sources = collections.defaultdict(dict)
         self._order = []
 
     def _key(self, repo_name, original_ref, filename):
@@ -30,8 +33,8 @@ class SourcePool(object):
         key = self._key(source.repo_name,
                         source.original_ref,
                         source.filename)
-        if key not in self._sources:
-            self._sources[key] = source
+        if key not in self._sources or source.name not in self._sources[key]:
+            self._sources[key][source.name] = source
             self._order.append(source)
 
     def lookup(self, repo_name, original_ref, filename):
@@ -42,7 +45,7 @@ class SourcePool(object):
         '''
 
         key = self._key(repo_name, original_ref, filename)
-        return self._sources[key]
+        return self._sources[key].values()
 
     def __iter__(self):
         '''Iterate over sources in the pool, in the order they were added.'''
diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py
index 0126b4d9..bfe0a716 100644
--- a/morphlib/stagingarea.py
+++ b/morphlib/stagingarea.py
@@ -65,8 +65,8 @@ class StagingArea(object):
         os.makedirs(dirname)
 
     def _dir_for_source(self, source, suffix):
-        basename = '%s.%s' % (str(source.morphology['name']), suffix)
-        dirname = os.path.join(self.dirname, basename)
+        dirname = os.path.join(self.dirname,
+                               '%s.%s' % (str(source.name), suffix))
         self._mkdir(dirname)
         return dirname