Build per-source rather than per-artifact

6 files changed, 314 insertions, 297 deletions

diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py
index a60a8989..5deb25b7 100644
--- a/morphlib/artifactresolver.py
+++ b/morphlib/artifactresolver.py
@@ -149,7 +149,7 @@ class ArtifactResolver(object):
                         if sta_name in stratum_source.artifacts:
                             stratum_artifact = \
                                 stratum_source.artifacts[sta_name]
-                            system.add_dependency(stratum_artifact)
+                            source.add_dependency(stratum_artifact)
                             artifacts.append(stratum_artifact)
 
                 queue.append(stratum_source)
@@ -182,10 +182,10 @@ class ArtifactResolver(object):
                     artifacts.append(other_stratum)
 
                     for stratum in strata:
-                        if other_stratum.depends_on(stratum):
+                        if other_source.depends_on(stratum):
                             raise MutualDependencyError(stratum, other_stratum)
 
-                        stratum.add_dependency(other_stratum)
+                        source.add_dependency(other_stratum)
 
                 queue.append(other_source)
 
@@ -208,12 +208,9 @@ class ArtifactResolver(object):
 
             build_depends = info.get('build-depends', None)
 
-            for ca_name in chunk_source.split_rules.artifacts:
-                chunk_artifact = chunk_source.artifacts[ca_name]
-
-                # Add our stratum's build depends as dependencies of this chunk
-                for other_stratum in stratum_build_depends:
-                    chunk_artifact.add_dependency(other_stratum)
+            # Add our stratum's build depends as dependencies of this chunk
+            for other_stratum in stratum_build_depends:
+                chunk_source.add_dependency(other_stratum)
 
             # Add dependencies between chunks mentioned in this stratum
             for name in build_depends: # pragma: no cover
@@ -222,22 +219,19 @@ class ArtifactResolver(object):
                         source, info['name'], name)
                 other_artifacts = name_to_processed_artifacts[name]
                 for other_artifact in other_artifacts:
-                    for ca_name in chunk_source.split_rules.artifacts:
-                        chunk_artifact = chunk_source.artifacts[ca_name]
-                        chunk_artifact.add_dependency(other_artifact)
+                    chunk_source.add_dependency(other_artifact)
 
             # Add build dependencies between our stratum's artifacts
             # and the chunk artifacts produced by this stratum.
             matches, overlaps, unmatched = source.split_rules.partition(
                     ((chunk_name, ca_name) for ca_name
                      in chunk_source.split_rules.artifacts))
-            for stratum in strata:
-                for (chunk_name, ca_name) in matches[stratum.name]:
-                    chunk_artifact = chunk_source.artifacts[ca_name]
-                    stratum.add_dependency(chunk_artifact)
-                    # Only return chunks required to build strata we need
-                    if chunk_artifact not in artifacts:
-                        artifacts.append(chunk_artifact)
+            for (chunk_name, ca_name) in matches[source.name]:
+                chunk_artifact = chunk_source.artifacts[ca_name]
+                source.add_dependency(chunk_artifact)
+                # Only return chunks required to build strata we need
+                if chunk_artifact not in artifacts:
+                    artifacts.append(chunk_artifact)
 
 
             # Add these chunks to the processed artifacts, so other
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index 436e23eb..c8d9930c 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -14,6 +14,7 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 
+import itertools
 import os
 import shutil
 import logging
@@ -175,9 +176,9 @@ class BuildCommand(object):
 
         self.app.status(msg='Computing cache keys', chatty=True)
         ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env)
-        for artifact in artifacts:
-            artifact.cache_key = ckc.compute_key(artifact)
-            artifact.cache_id = ckc.get_cache_id(artifact)
+        for source in set(a.source for a in artifacts):
+            source.cache_key = ckc.compute_key(source)
+            source.cache_id = ckc.get_cache_id(source)
 
         root_artifact.build_env = build_env
         return root_artifact
@@ -210,7 +211,7 @@ class BuildCommand(object):
             if src.morphology['kind'] == 'stratum':
                 name = src.name
                 ref = src.sha1[:7]
-                self.app.status(msg='Stratum [%(name)s] version is %(ref)s', 
+                self.app.status(msg='Stratum [%(name)s] version is %(ref)s',
                                 name=name, ref=ref)
                 if name in stratum_names:
                     raise morphlib.Error(
@@ -251,78 +252,94 @@ class BuildCommand(object):
 
     def _find_root_artifacts(self, artifacts):
         '''Find all the root artifacts among a set of artifacts in a DAG.
-        
+
         It would be nice if the ArtifactResolver would return its results in a
         more useful order to save us from needing to do this -- the root object
         is known already since that's the one the user asked us to build.
-        
+
         '''
 
-        return [a for a in artifacts if not a.dependents]
+        return [a for a in artifacts if not a.dependent_sources]
+
+    @staticmethod
+    def get_ordered_sources(artifacts):
+        ordered_sources = []
+        known_sources = set()
+        for artifact in artifacts:
+            if artifact.source not in known_sources:
+                known_sources.add(artifact.source)
+                yield artifact.source
 
     def build_in_order(self, root_artifact):
         '''Build everything specified in a build order.'''
 
-        self.app.status(msg='Building a set of artifacts', chatty=True)
+        self.app.status(msg='Building a set of sources', chatty=True)
         build_env = root_artifact.build_env
-        artifacts = root_artifact.walk()
+        ordered_sources = list(self.get_ordered_sources(root_artifact.walk()))
         old_prefix = self.app.status_prefix
-        for i, a in enumerate(artifacts):
+        for i, s in enumerate(ordered_sources):
             self.app.status_prefix = (
                 old_prefix + '[Build %(index)d/%(total)d] [%(name)s] ' % {
                     'index': (i+1),
-                    'total': len(artifacts),
-                    'name': a.name,
+                    'total': len(ordered_sources),
+                    'name': s.name,
                 })
 
-            self.cache_or_build_artifact(a, build_env)
-
-            self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s',
-                            kind=a.source.morphology['kind'], name=a.name,
-                            cachepath=self.lac.artifact_filename(a),
-                            chatty=(a.source.morphology['kind'] != "system"))
+            self.cache_or_build_source(s, build_env)
 
         self.app.status_prefix = old_prefix
 
-    def cache_or_build_artifact(self, artifact, build_env):
-        '''Make the built artifact available in the local cache.
+    def cache_or_build_source(self, source, build_env):
+        '''Make artifacts of the built source available in the local cache.
 
         This can be done by retrieving from a remote artifact cache, or if
-        that doesn't work for some reason, by building the artifact locally.
+        that doesn't work for some reason, by building the source locally.
 
         '''
+        artifacts = source.artifacts.values()
         if self.rac is not None:
             try:
-                self.cache_artifacts_locally([artifact])
+                self.cache_artifacts_locally(artifacts)
             except morphlib.remoteartifactcache.GetError:
                 # Error is logged by the RemoteArtifactCache object.
                 pass
 
-        if not self.lac.has(artifact):
-            self.build_artifact(artifact, build_env)
+        if any(not self.lac.has(artifact) for artifact in artifacts):
+            self.build_source(source, build_env)
+
+        for a in artifacts:
+            self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s',
+                            kind=source.morphology['kind'], name=a.name,
+                            cachepath=self.lac.artifact_filename(a),
+                            chatty=(source.morphology['kind'] != "system"))
 
-    def build_artifact(self, artifact, build_env):
-        '''Build one artifact.
+    def build_source(self, source, build_env):
+        '''Build all artifacts for one source.
 
         All the dependencies are assumed to be built and available
         in either the local or remote cache already.
 
         '''
         self.app.status(msg='Building %(kind)s %(name)s',
-                        name=artifact.name,
-                        kind=artifact.source.morphology['kind'])
-
-        self.fetch_sources(artifact)
-        deps = self.get_recursive_deps(artifact)
+                        name=source.name,
+                        kind=source.morphology['kind'])
+
+        self.fetch_sources(source)
+        # TODO: Make an artifact.walk() that takes multiple root artifacts.
+        # as this does a walk for every artifact. This was the status
+        # quo before build logic was made to work per-source, but we can
+        # now do better.
+        deps = self.get_recursive_deps(source.artifacts.values())
         self.cache_artifacts_locally(deps)
 
         use_chroot = False
         setup_mounts = False
-        if artifact.source.morphology['kind'] == 'chunk':
-            build_mode = artifact.source.build_mode
-            extra_env = {'PREFIX': artifact.source.prefix}
+        if source.morphology['kind'] == 'chunk':
+            build_mode = source.build_mode
+            extra_env = {'PREFIX': source.prefix}
 
-            dep_prefix_set = artifact.get_dependency_prefix_set()
+            dep_prefix_set = set(a.source.prefix for a in deps
+                                 if a.source.morphology['kind'] == 'chunk')
             extra_path = [os.path.join(d, 'bin') for d in dep_prefix_set]
 
             if build_mode not in ['bootstrap', 'staging', 'test']:
@@ -340,37 +357,44 @@ class BuildCommand(object):
                                                     extra_env=extra_env,
                                                     extra_path=extra_path)
             try:
-                self.install_dependencies(staging_area, deps, artifact)
+                self.install_dependencies(staging_area, deps, source)
             except BaseException:
                 staging_area.abort()
                 raise
         else:
             staging_area = self.create_staging_area(build_env, False)
 
-        self.build_and_cache(staging_area, artifact, setup_mounts)
+        self.build_and_cache(staging_area, source, setup_mounts)
         self.remove_staging_area(staging_area)
 
-    def get_recursive_deps(self, artifact):
-        return artifact.walk()[:-1]
+    def get_recursive_deps(self, artifacts):
+        deps = set()
+        ordered_deps = []
+        for artifact in artifacts:
+            for dep in artifact.walk():
+                if dep not in deps and dep not in artifacts:
+                    deps.add(dep)
+                    ordered_deps.append(dep)
+        return ordered_deps
 
-    def fetch_sources(self, artifact):
+    def fetch_sources(self, source):
         '''Update the local git repository cache with the sources.'''
 
-        repo_name = artifact.source.repo_name
+        repo_name = source.repo_name
         if self.app.settings['no-git-update']:
             self.app.status(msg='Not updating existing git repository '
                                 '%(repo_name)s '
                                 'because of no-git-update being set',
                             chatty=True,
                             repo_name=repo_name)
-            artifact.source.repo = self.lrc.get_repo(repo_name)
+            source.repo = self.lrc.get_repo(repo_name)
             return
 
         if self.lrc.has_repo(repo_name):
-            artifact.source.repo = self.lrc.get_repo(repo_name)
+            source.repo = self.lrc.get_repo(repo_name)
             try:
-                sha1 = artifact.source.sha1
-                artifact.source.repo.resolve_ref(sha1)
+                sha1 = source.sha1
+                source.repo.resolve_ref(sha1)
                 self.app.status(msg='Not updating git repository '
                                     '%(repo_name)s because it '
                                     'already contains sha1 %(sha1)s',
@@ -379,17 +403,17 @@ class BuildCommand(object):
             except morphlib.cachedrepo.InvalidReferenceError:
                 self.app.status(msg='Updating %(repo_name)s',
                                 repo_name=repo_name)
-                artifact.source.repo.update()
+                source.repo.update()
         else:
             self.app.status(msg='Cloning %(repo_name)s',
                             repo_name=repo_name)
-            artifact.source.repo = self.lrc.cache_repo(repo_name)
+            source.repo = self.lrc.cache_repo(repo_name)
 
         # Update submodules.
         done = set()
         self.app.cache_repo_and_submodules(
-            self.lrc, artifact.source.repo.url,
-            artifact.source.sha1, done)
+            self.lrc, source.repo.url,
+            source.sha1, done)
 
     def cache_artifacts_locally(self, artifacts):
         '''Get artifacts missing from local cache from remote cache.'''
@@ -455,14 +479,25 @@ class BuildCommand(object):
 
     # Nasty hack to avoid installing chunks built in 'bootstrap' mode in a
     # different stratum when constructing staging areas.
-    def is_stratum(self, a):
-        return a.source.morphology['kind'] == 'stratum'
+    # TODO: make nicer by having chunk morphs keep a reference to the
+    #       stratum they were in
+    def in_same_stratum(self, s1, s2):
+        '''Checks whether two chunk sources are from the same stratum.
 
-    def in_same_stratum(self, a, b):
-        return len(filter(self.is_stratum, a.dependencies)) == \
-               len(filter(self.is_stratum, b.dependencies))
+        In the absence of morphologies tracking where they came from,
+        this checks whether both sources are depended on by artifacts
+        that belong to sources which have the same morphology.
 
-    def install_dependencies(self, staging_area, artifacts, target_artifact):
+        '''
+        def dependent_stratum_morphs(source):
+            dependent_sources = set(itertools.chain.from_iterable(
+                a.dependent_sources for a in source.artifacts.itervalues()))
+            dependent_strata = set(s for s in dependent_sources
+                                   if s.morphology['kind'] == 'stratum')
+            return set(s.morphology for s in dependent_strata)
+        return dependent_stratum_morphs(s1) == dependent_stratum_morphs(s2)
+
+    def install_dependencies(self, staging_area, artifacts, target_source):
         '''Install chunk artifacts into staging area.
 
         We only ever care about chunk artifacts as build dependencies,
@@ -477,29 +512,29 @@ class BuildCommand(object):
             if artifact.source.morphology['kind'] != 'chunk':
                 continue
             if artifact.source.build_mode == 'bootstrap':
-               if not self.in_same_stratum(artifact, target_artifact):
+               if not self.in_same_stratum(artifact.source, target_source):
                     continue
             self.app.status(
                 msg='Installing chunk %(chunk_name)s from cache %(cache)s',
                 chunk_name=artifact.name,
-                cache=artifact.cache_key[:7],
+                cache=artifact.source.cache_key[:7],
                 chatty=True)
             handle = self.lac.get(artifact)
             staging_area.install_artifact(handle)
 
-        if target_artifact.source.build_mode == 'staging':
+        if target_source.build_mode == 'staging':
             morphlib.builder2.ldconfig(self.app.runcmd, staging_area.dirname)
 
-    def build_and_cache(self, staging_area, artifact, setup_mounts):
-        '''Build an artifact and put it into the local artifact cache.'''
+    def build_and_cache(self, staging_area, source, setup_mounts):
+        '''Build a source and put its artifacts into the local cache.'''
 
         self.app.status(msg='Starting actual build: %(name)s '
                             '%(sha1)s',
-                        name=artifact.name, sha1=artifact.source.sha1[:7])
+                        name=source.name, sha1=source.sha1[:7])
         builder = morphlib.builder2.Builder(
             self.app, staging_area, self.lac, self.rac, self.lrc,
             self.app.settings['max-jobs'], setup_mounts)
-        return builder.build_and_cache(artifact)
+        return builder.build_and_cache(source)
 
 class InitiatorBuildCommand(BuildCommand):
 
diff --git a/morphlib/builder2.py b/morphlib/builder2.py
index 681ad6be..20cae225 100644
--- a/morphlib/builder2.py
+++ b/morphlib/builder2.py
@@ -162,15 +162,15 @@ def get_stratum_files(f, lac):  # pragma: no cover
         cf.close()
 
 
-def get_overlaps(artifact, constituents, lac):  # pragma: no cover
+def get_overlaps(source, constituents, lac):  # pragma: no cover
     # check whether strata overlap
     installed = defaultdict(set)
     for dep in constituents:
         handle = lac.get(dep)
-        if artifact.source.morphology['kind'] == 'stratum':
+        if source.morphology['kind'] == 'stratum':
             for filename in get_chunk_files(handle):
                 installed[filename].add(dep)
-        elif artifact.source.morphology['kind'] == 'system':
+        elif source.morphology['kind'] == 'system':
             for filename in get_stratum_files(handle, lac):
                 installed[filename].add(dep)
         handle.close()
@@ -207,13 +207,13 @@ class BuilderBase(object):
     '''Base class for building artifacts.'''
 
     def __init__(self, app, staging_area, local_artifact_cache,
-                 remote_artifact_cache, artifact, repo_cache, max_jobs,
+                 remote_artifact_cache, source, repo_cache, max_jobs,
                  setup_mounts):
         self.app = app
         self.staging_area = staging_area
         self.local_artifact_cache = local_artifact_cache
         self.remote_artifact_cache = remote_artifact_cache
-        self.artifact = artifact
+        self.source = source
         self.repo_cache = repo_cache
         self.max_jobs = max_jobs
         self.build_watch = morphlib.stopwatch.Stopwatch()
@@ -233,13 +233,13 @@ class BuilderBase(object):
 
         logging.debug('Writing metadata to the cache')
         with self.local_artifact_cache.put_source_metadata(
-                self.artifact.source, self.artifact.cache_key,
+                self.source, self.source.cache_key,
                 'meta') as f:
             json.dump(meta, f, indent=4, sort_keys=True,
                       encoding='unicode-escape')
             f.write('\n')
 
-    def create_metadata(self, artifact_name, contents=[]):
+    def create_metadata(self, artifact_name, contents=[]): # pragma: no cover
         '''Create metadata to artifact to allow it to be reproduced later.
 
         The metadata is represented as a dict, which later on will be
@@ -247,20 +247,20 @@ class BuilderBase(object):
 
         '''
 
-        assert isinstance(self.artifact.source.repo,
+        assert isinstance(self.source.repo,
                           morphlib.cachedrepo.CachedRepo)
         meta = {
             'artifact-name': artifact_name,
-            'source-name': self.artifact.source.morphology['name'],
-            'kind': self.artifact.source.morphology['kind'],
-            'description': self.artifact.source.morphology['description'],
-            'repo': self.artifact.source.repo.url,
-            'repo-alias': self.artifact.source.repo_name,
-            'original_ref': self.artifact.source.original_ref,
-            'sha1': self.artifact.source.sha1,
-            'morphology': self.artifact.source.filename,
-            'cache-key': self.artifact.cache_key,
-            'cache-id': self.artifact.cache_id,
+            'source-name': self.source.name,
+            'kind': self.source.morphology['kind'],
+            'description': self.source.morphology['description'],
+            'repo': self.source.repo.url,
+            'repo-alias': self.source.repo_name,
+            'original_ref': self.source.original_ref,
+            'sha1': self.source.sha1,
+            'morphology': self.source.filename,
+            'cache-key': self.source.cache_key,
+            'cache-id': self.source.cache_id,
             'morph-version': {
                 'ref': morphlib.gitversion.ref,
                 'tree': morphlib.gitversion.tree,
@@ -279,7 +279,8 @@ class BuilderBase(object):
             os.makedirs(dirname)
         return open(filename, mode)
 
-    def write_metadata(self, instdir, artifact_name, contents=[]):
+    def write_metadata(self, instdir, artifact_name,
+                       contents=[]): # pragma: no cover
         '''Write the metadata for an artifact.
 
         The file will be located under the ``baserock`` directory under
@@ -308,7 +309,7 @@ class ChunkBuilder(BuilderBase):
 
     def create_devices(self, destdir): # pragma: no cover
         '''Creates device nodes if the morphology specifies them'''
-        morphology = self.artifact.source.morphology
+        morphology = self.source.morphology
         perms_mask = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO
         if 'devices' in morphology and morphology['devices'] is not None:
             for dev in morphology['devices']:
@@ -332,14 +333,14 @@ class ChunkBuilder(BuilderBase):
         with self.build_watch('overall-build'):
 
             builddir, destdir = self.staging_area.chroot_open(
-                self.artifact.source, self.setup_mounts)
+                self.source, self.setup_mounts)
 
             stdout = (self.app.output
                 if self.app.settings['build-log-on-stdout'] else None)
 
             cache = self.local_artifact_cache
             logpath = cache.get_source_metadata_filename(
-                self.artifact.source, self.artifact.cache_key, 'build-log')
+                self.source, self.source.cache_key, 'build-log')
 
             _, temppath = tempfile.mkstemp(dir=os.path.dirname(logpath))
 
@@ -375,7 +376,7 @@ class ChunkBuilder(BuilderBase):
 
     def run_commands(self, builddir, destdir,
                      logfilepath, stdout=None):  # pragma: no cover
-        m = self.artifact.source.morphology
+        m = self.source.morphology
         bs = morphlib.buildsystem.lookup_build_system(m['build-system'])
 
         relative_builddir = self.staging_area.relative(builddir)
@@ -407,7 +408,7 @@ class ChunkBuilder(BuilderBase):
 
                 for cmd in cmds:
                     if in_parallel:
-                        max_jobs = self.artifact.source.morphology['max-jobs']
+                        max_jobs = self.source.morphology['max-jobs']
                         if max_jobs is None:
                             max_jobs = self.max_jobs
                         extra_env['MAKEFLAGS'] = '-j%s' % max_jobs
@@ -474,7 +475,7 @@ class ChunkBuilder(BuilderBase):
     def assemble_chunk_artifacts(self, destdir):  # pragma: no cover
         built_artifacts = []
         filenames = []
-        source = self.artifact.source
+        source = self.source
         split_rules = source.split_rules
         morphology = source.morphology
         sys_tag = 'system-integration'
@@ -533,7 +534,7 @@ class ChunkBuilder(BuilderBase):
         return built_artifacts
 
     def get_sources(self, srcdir):  # pragma: no cover
-        s = self.artifact.source
+        s = self.source
         extract_sources(self.app, self.repo_cache, s.repo, s.sha1, srcdir)
 
 
@@ -547,42 +548,42 @@ class StratumBuilder(BuilderBase):
 
     def build_and_cache(self):  # pragma: no cover
         with self.build_watch('overall-build'):
-            constituents = [d for d in self.artifact.dependencies
+            constituents = [d for d in self.source.dependencies
                             if self.is_constituent(d)]
 
             # the only reason the StratumBuilder has to download chunks is to
             # check for overlap now that strata are lists of chunks
             with self.build_watch('check-chunks'):
-                # download the chunk artifact if necessary
-                download_depends(constituents,
-                                 self.local_artifact_cache,
-                                 self.remote_artifact_cache)
-                # check for chunk overlaps
-                overlaps = get_overlaps(self.artifact, constituents,
-                                        self.local_artifact_cache)
-                if len(overlaps) > 0:
-                    logging.warning('Overlaps in stratum artifact %s detected'
-                                    % self.artifact.name)
-                    log_overlaps(overlaps)
-                    self.app.status(msg='Overlaps in stratum artifact '
-                                        '%(stratum_name)s detected',
-                                    stratum_name=self.artifact.name,
-                                    error=True)
-                    write_overlap_metadata(self.artifact, overlaps,
-                                           self.local_artifact_cache)
+                for a_name, a in self.source.artifacts.iteritems():
+                    # download the chunk artifact if necessary
+                    download_depends(constituents,
+                                     self.local_artifact_cache,
+                                     self.remote_artifact_cache)
+                    # check for chunk overlaps
+                    overlaps = get_overlaps(self.source, constituents,
+                                            self.local_artifact_cache)
+                    if len(overlaps) > 0:
+                        logging.warning(
+                            'Overlaps in stratum artifact %s detected' %a_name)
+                        log_overlaps(overlaps)
+                        self.app.status(msg='Overlaps in stratum artifact '
+                                            '%(stratum_name)s detected',
+                                        stratum_name=a_name, error=True)
+                        write_overlap_metadata(a, overlaps,
+                                               self.local_artifact_cache)
 
             with self.build_watch('create-chunk-list'):
                 lac = self.local_artifact_cache
-                meta = self.create_metadata(self.artifact.name,
-                                            [x.name for x in constituents])
-                with lac.put_artifact_metadata(self.artifact, 'meta')  as f:
-                    json.dump(meta, f, indent=4, sort_keys=True,
-                              encoding='unicode-escape')
-                with self.local_artifact_cache.put(self.artifact) as f:
-                    json.dump([c.basename() for c in constituents], f,
-                              encoding='unicode-escape')
+                for a_name, a in self.source.artifacts.iteritems():
+                    meta = self.create_metadata(
+                        a_name,
+                        [x.name for x in constituents])
+                    with lac.put_artifact_metadata(a, 'meta')  as f:
+                        json.dump(meta, f, indent=4, sort_keys=True)
+                    with self.local_artifact_cache.put(a) as f:
+                        json.dump([c.basename() for c in constituents], f)
         self.save_build_times()
-        return [self.artifact]
+        return self.source.artifacts.values()
 
 
 class SystemBuilder(BuilderBase):  # pragma: no cover
@@ -596,43 +597,42 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
 
     def build_and_cache(self):
         self.app.status(msg='Building system %(system_name)s',
-                        system_name=self.artifact.source.morphology['name'])
+                        system_name=self.source.name)
 
         with self.build_watch('overall-build'):
-            arch = self.artifact.source.morphology['arch']
-
-            rootfs_name = self.artifact.source.morphology['name']
-            handle = self.local_artifact_cache.put(self.artifact)
-
-            try:
-                fs_root = self.staging_area.destdir(self.artifact.source)
-                self.unpack_strata(fs_root)
-                self.write_metadata(fs_root, rootfs_name)
-                self.run_system_integration_commands(fs_root)
-                unslashy_root = fs_root[1:]
-                def uproot_info(info):
-                    info.name = relpath(info.name, unslashy_root)
-                    if info.islnk():
-                        info.linkname = relpath(info.linkname,
-                                                unslashy_root)
-                    return info
-                artiname = self.artifact.source.morphology['name']
-                tar = tarfile.open(fileobj=handle, mode="w", name=artiname)
-                self.app.status(msg='Constructing tarball of root filesystem',
-                                chatty=True)
-                tar.add(fs_root, recursive=True, filter=uproot_info)
-                tar.close()
-            except BaseException, e:
-                logging.error(traceback.format_exc())
-                self.app.status(msg='Error while building system',
-                                error=True)
-                handle.abort()
-                raise
-
-            handle.close()
+            arch = self.source.morphology['arch']
+
+            for a_name, artifact in self.source.artifacts.iteritems():
+                handle = self.local_artifact_cache.put(artifact)
+
+                try:
+                    fs_root = self.staging_area.destdir(self.source)
+                    self.unpack_strata(fs_root)
+                    self.write_metadata(fs_root, a_name)
+                    self.run_system_integration_commands(fs_root)
+                    unslashy_root = fs_root[1:]
+                    def uproot_info(info):
+                        info.name = relpath(info.name, unslashy_root)
+                        if info.islnk():
+                            info.linkname = relpath(info.linkname,
+                                                    unslashy_root)
+                        return info
+                    tar = tarfile.open(fileobj=handle, mode="w", name=a_name)
+                    self.app.status(msg='Constructing tarball of rootfs',
+                                    chatty=True)
+                    tar.add(fs_root, recursive=True, filter=uproot_info)
+                    tar.close()
+                except BaseException as e:
+                    logging.error(traceback.format_exc())
+                    self.app.status(msg='Error while building system',
+                                    error=True)
+                    handle.abort()
+                    raise
+                else:
+                    handle.close()
 
         self.save_build_times()
-        return [self.artifact]
+        return self.source.artifacts.itervalues()
 
     def unpack_one_stratum(self, stratum_artifact, target):
         '''Unpack a single stratum into a target directory'''
@@ -658,37 +658,37 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
         self.app.status(msg='Unpacking strata to %(path)s',
                         path=path, chatty=True)
         with self.build_watch('unpack-strata'):
-            # download the stratum artifacts if necessary
-            download_depends(self.artifact.dependencies,
-                             self.local_artifact_cache,
-                             self.remote_artifact_cache,
-                             ('meta',))
-
-            # download the chunk artifacts if necessary
-            for stratum_artifact in self.artifact.dependencies:
-                f = self.local_artifact_cache.get(stratum_artifact)
-                chunks = [ArtifactCacheReference(a)
-                          for a in json.load(f, encoding='unicode-escape')]
-                download_depends(chunks,
+            for a_name, a in self.source.artifacts.iteritems():
+                # download the stratum artifacts if necessary
+                download_depends(self.source.dependencies,
                                  self.local_artifact_cache,
-                                 self.remote_artifact_cache)
-                f.close()
-
-            # check whether the strata overlap
-            overlaps = get_overlaps(self.artifact, self.artifact.dependencies,
-                                    self.local_artifact_cache)
-            if len(overlaps) > 0:
-                self.app.status(msg='Overlaps in system artifact '
-                                    '%(artifact_name)s detected',
-                                artifact_name=self.artifact.name,
-                                error=True)
-                log_overlaps(overlaps)
-                write_overlap_metadata(self.artifact, overlaps,
-                                       self.local_artifact_cache)
-
-            # unpack it from the local artifact cache
-            for stratum_artifact in self.artifact.dependencies:
-                self.unpack_one_stratum(stratum_artifact, path)
+                                 self.remote_artifact_cache,
+                                 ('meta',))
+
+                # download the chunk artifacts if necessary
+                for stratum_artifact in self.source.dependencies:
+                    f = self.local_artifact_cache.get(stratum_artifact)
+                    chunks = [ArtifactCacheReference(c) for c in json.load(f)]
+                    download_depends(chunks,
+                                     self.local_artifact_cache,
+                                     self.remote_artifact_cache)
+                    f.close()
+
+                # check whether the strata overlap
+                overlaps = get_overlaps(self.source, self.source.dependencies,
+                                        self.local_artifact_cache)
+                if len(overlaps) > 0:
+                    self.app.status(msg='Overlaps in system artifact '
+                                        '%(artifact_name)s detected',
+                                    artifact_name=a_name,
+                                    error=True)
+                    log_overlaps(overlaps)
+                    write_overlap_metadata(a, overlaps,
+                                           self.local_artifact_cache)
+
+                # unpack it from the local artifact cache
+                for stratum_artifact in self.source.dependencies:
+                    self.unpack_one_stratum(stratum_artifact, path)
 
             ldconfig(self.app.runcmd, path)
 
@@ -779,15 +779,15 @@ class Builder(object):  # pragma: no cover
         self.max_jobs = max_jobs
         self.setup_mounts = setup_mounts
 
-    def build_and_cache(self, artifact):
-        kind = artifact.source.morphology['kind']
+    def build_and_cache(self, source):
+        kind = source.morphology['kind']
         o = self.classes[kind](self.app, self.staging_area,
                                self.local_artifact_cache,
-                               self.remote_artifact_cache, artifact,
+                               self.remote_artifact_cache, source,
                                self.repo_cache, self.max_jobs,
                                self.setup_mounts)
         self.app.status(msg='Builder.build: artifact %s with %s' %
-                       (artifact.name, repr(o)),
+                       (source.name, repr(o)),
                        chatty=True)
         built_artifacts = o.build_and_cache()
         self.app.status(msg='Builder.build: done',
diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py
index a9f5aabe..c3a01b9e 100644
--- a/morphlib/cachekeycomputer.py
+++ b/morphlib/cachekeycomputer.py
@@ -32,16 +32,15 @@ class CacheKeyComputer(object):
                 "USER", "USERNAME"]
         return dict([(k, env[k]) for k in keys])
 
-    def compute_key(self, artifact):
+    def compute_key(self, source):
         try:
-            ret = self._hashed[artifact]
-            return ret
+            return self._hashed[source]
         except KeyError:
-            ret = self._hash_id(self.get_cache_id(artifact))
-            self._hashed[artifact] = ret
-            logging.debug('computed cache key %s for artifact %s from source ',
-                          ret, (artifact.source.repo_name,
-                           artifact.source.sha1, artifact.source.filename))
+            ret = self._hash_id(self.get_cache_id(source))
+            self._hashed[source] = ret
+            logging.debug(
+                'computed cache key %s for artifact %s from source ',
+                 ret, (source.repo_name, source.sha1, source.filename))
             return ret
 
     def _hash_id(self, cache_id):
@@ -71,47 +70,47 @@ class CacheKeyComputer(object):
         for item in tup:
             self._hash_thing(sha, item)
 
-    def get_cache_id(self, artifact):
+    def get_cache_id(self, source):
         try:
-            ret = self._calculated[artifact]
+            ret = self._calculated[source]
             return ret
         except KeyError:
-            cacheid = self._calculate(artifact)
-            self._calculated[artifact] = cacheid
+            cacheid = self._calculate(source)
+            self._calculated[source] = cacheid
             return cacheid
 
-    def _calculate(self, artifact):
+    def _calculate(self, source):
         keys = {
             'env': self._filterenv(self._build_env.env),
-            'kids': [{'artifact': a.name, 'cache-key': self.compute_key(a)}
-                     for a in artifact.dependencies],
-            'metadata-version': 1 # bump if /baserock metadata format changes
+            'kids': [{'artifact': a.name,
+                      'cache-key': self.compute_key(a.source)}
+                     for a in source.dependencies],
+            'metadata-version': 1
         }
 
-        kind = artifact.source.morphology['kind']
+        morphology = source.morphology
+        kind = morphology['kind']
         if kind == 'chunk':
-            keys['build-mode'] = artifact.source.build_mode
-            keys['prefix'] = artifact.source.prefix
-            keys['tree'] = artifact.source.tree
+            keys['build-mode'] = source.build_mode
+            keys['prefix'] = source.prefix
+            keys['tree'] = source.tree
             keys['split-rules'] = [(a, [rgx.pattern for rgx in r._regexes])
-                                   for (a, r) in artifact.source.split_rules]
+                                   for (a, r) in source.split_rules]
 
             # Include morphology contents, since it doesn't always come
             # from the source tree
-            morphology = artifact.source.morphology
+            keys['devices'] = morphology.get('devices')
+            keys['max-jobs'] = morphology.get('max-jobs')
+            keys['system-integration'] = morphology.get('system-integration',
+                                                        {})
+            # products is omitted as they are part of the split-rules
             # include {pre-,,post-}{configure,build,test,install}-commands
             # in morphology key
             for prefix in ('pre-', '', 'post-'):
                 for cmdtype in ('configure', 'build', 'test', 'install'):
                     cmd_field = prefix + cmdtype + '-commands'
                     keys[cmd_field] = morphology[cmd_field]
-            keys['devices'] = morphology.get('devices')
-            keys['max-jobs'] = morphology.get('max-jobs')
-            keys['system-integration'] = morphology.get('system-integration',
-                                                        {})
-            # products is omitted as they are part of the split-rules
         elif kind in ('system', 'stratum'):
-            morphology = artifact.source.morphology
             morph_dict = dict((k, morphology[k]) for k in morphology.keys())
 
             # Disregard all fields of a morphology that aren't important
diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py
index 95e991f4..7b53a4a5 100644
--- a/morphlib/plugins/cross-bootstrap_plugin.py
+++ b/morphlib/plugins/cross-bootstrap_plugin.py
@@ -58,38 +58,39 @@ class BootstrapSystemBuilder(morphlib.builder2.BuilderBase):
 
     def build_and_cache(self):
         with self.build_watch('overall-build'):
-            handle = self.local_artifact_cache.put(self.artifact)
-            fs_root = self.staging_area.destdir(self.artifact.source)
-            try:
-                self.unpack_binary_chunks(fs_root)
-                self.unpack_sources(fs_root)
-                self.write_build_script(fs_root)
-                system_name = self.artifact.source.morphology['name']
-                self.create_tarball(handle, fs_root, system_name)
-            except BaseException, e:
-                logging.error(traceback.format_exc())
-                self.app.status(msg='Error while building bootstrap image',
-                                error=True)
-                handle.abort()
-                raise
-
-            handle.close()
+            for system_name, artifact in self.source.artifacts.iteritems():
+                handle = self.local_artifact_cache.put(artifact)
+                fs_root = self.staging_area.destdir(self.source)
+                try:
+                    self.unpack_binary_chunks(fs_root)
+                    self.unpack_sources(fs_root)
+                    self.write_build_script(fs_root)
+                    self.create_tarball(handle, fs_root, system_name)
+                except BaseException, e:
+                    logging.error(traceback.format_exc())
+                    self.app.status(msg='Error while building bootstrap image',
+                                    error=True)
+                    handle.abort()
+                    raise
+
+                handle.close()
 
         self.save_build_times()
-        return [self.artifact]
+        return self.source.artifacts.items()
 
     def unpack_binary_chunks(self, dest):
         cache = self.local_artifact_cache
-        for chunk_artifact in self.artifact.source.cross_chunks:
-            with cache.get(chunk_artifact) as chunk_file:
-                try:
-                    morphlib.bins.unpack_binary_from_file(chunk_file, dest)
-                except BaseException, e:
-                    self.app.status(
-                        msg='Error unpacking binary chunk %(name)s',
-                        name=chunk_artifact.name,
-                        error=True)
-                    raise
+        for chunk_source in self.source.cross_sources:
+            for chunk_artifact in chunk_source.artifacts.itervalues():
+                with cache.get(chunk_artifact) as chunk_file:
+                    try:
+                        morphlib.bins.unpack_binary_from_file(chunk_file, dest)
+                    except BaseException, e:
+                        self.app.status(
+                            msg='Error unpacking binary chunk %(name)s',
+                            name=chunk_artifact.name,
+                            error=True)
+                        raise
 
     def unpack_sources(self, path):
         # Multiple chunks sources may be built from the same repo ('linux'
@@ -98,24 +99,18 @@ class BootstrapSystemBuilder(morphlib.builder2.BuilderBase):
         #
         # It might be neater to build these as "source artifacts" individually,
         # but that would waste huge amounts of space in the artifact cache.
-        for a in self.artifact.walk():
-            if a in self.artifact.source.cross_chunks:
-                continue
-            if a.source.morphology['kind'] != 'chunk':
-                continue
-
-            escaped_source = escape_source_name(a.source)
+        for s in self.source.native_sources:
+            escaped_source = escape_source_name(s)
             source_dir = os.path.join(path, 'src', escaped_source)
             if not os.path.exists(source_dir):
                 os.makedirs(source_dir)
                 morphlib.builder2.extract_sources(
-                    self.app, self.repo_cache, a.source.repo, a.source.sha1,
-                    source_dir)
+                    self.app, self.repo_cache, s.repo, s.sha1, source_dir)
 
-            name = a.source.morphology['name']
+            name = s.name
             chunk_script = os.path.join(path, 'src', 'build-%s' % name)
             with morphlib.savefile.SaveFile(chunk_script, 'w') as f:
-                self.write_chunk_build_script(a, f)
+                self.write_chunk_build_script(s, f)
             os.chmod(chunk_script, 0777)
 
     def write_build_script(self, path):
@@ -130,15 +125,8 @@ class BootstrapSystemBuilder(morphlib.builder2.BuilderBase):
                 if k != 'PATH':
                     f.write('export %s="%s"\n' % (k, v))
 
-            # FIXME: really, of course, we need to iterate the sources not the
-            # artifacts ... this will break when we have chunk splitting!
-            for a in self.artifact.walk():
-                if a in self.artifact.source.cross_chunks:
-                    continue
-                if a.source.morphology['kind'] != 'chunk':
-                    continue
-
-                name = a.source.morphology['name']
+            for s in self.source.native_sources:
+                name = s.name
                 f.write('\necho Building %s\n' % name)
                 f.write('mkdir /%s.inst\n' % name)
                 f.write('env DESTDIR=/%s.inst $SRCDIR/build-%s\n'
@@ -150,17 +138,17 @@ class BootstrapSystemBuilder(morphlib.builder2.BuilderBase):
             f.write(driver_footer)
         os.chmod(driver_script, 0777)
 
-    def write_chunk_build_script(self, chunk, f):
-        m = chunk.source.morphology
+    def write_chunk_build_script(self, source, f):
+        m = source.morphology
         f.write('#!/bin/sh\n')
         f.write('# Build script generated by morph\n')
         f.write('set -e\n')
         f.write('chunk_name=%s\n' % m['name'])
 
-        repo = escape_source_name(chunk.source)
+        repo = escape_source_name(source)
         f.write('cp -a $SRCDIR/%s $DESTDIR/$chunk_name.build\n' % repo)
         f.write('cd $DESTDIR/$chunk_name.build\n')
-        f.write('export PREFIX=%s\n' % chunk.source.prefix)
+        f.write('export PREFIX=%s\n' % source.prefix)
 
         bs = morphlib.buildsystem.lookup_build_system(m['build-system'])
 
@@ -280,35 +268,36 @@ class CrossBootstrapPlugin(cliapp.Plugin):
 
         # Calculate build order
         # This is basically a hacked version of BuildCommand.build_in_order()
-        artifacts = system_artifact.walk()
-        cross_chunks = []
-        native_chunks = []
-        for a in artifacts:
-            if a.source.morphology['kind'] == 'chunk':
-                if a.source.build_mode == 'bootstrap':
-                    cross_chunks.append(a)
+        sources = build_command.get_ordered_sources(system_artifact.walk())
+        cross_sources = []
+        native_sources = []
+        for s in sources:
+            if s.morphology['kind'] == 'chunk':
+                if s.build_mode == 'bootstrap':
+                    cross_sources.append(s)
                 else:
-                    native_chunks.append(a)
+                    native_sources.append(s)
 
-        if len(cross_chunks) == 0:
+        if len(cross_sources) == 0:
             raise morphlib.Error(
                 'Nothing to cross-compile. Only chunks built in \'bootstrap\' '
                 'mode can be cross-compiled.')
 
-        for i, a in enumerate(cross_chunks):
-            build_command.cache_or_build_artifact(a, build_env)
+        for s in cross_sources:
+            build_command.cache_or_build_source(s, build_env)
 
-        for i, a in enumerate(native_chunks):
-            build_command.fetch_sources(a)
+        for s in native_sources:
+            build_command.fetch_sources(s)
 
         # Install those to the output tarball ...
         self.app.status(msg='Building final bootstrap system image')
-        system_artifact.source.cross_chunks = cross_chunks
+        system_artifact.source.cross_sources = cross_sources
+        system_artifact.source.native_sources = native_sources
         staging_area = build_command.create_staging_area(
             build_env, use_chroot=False)
         builder = BootstrapSystemBuilder(
             self.app, staging_area, build_command.lac, build_command.rac,
-            system_artifact, build_command.lrc, 1, False)
+            system_artifact.source, build_command.lrc, 1, False)
         builder.build_and_cache()
 
         self.app.status(
diff --git a/morphlib/remoteartifactcache.py b/morphlib/remoteartifactcache.py
index 0f8edce8..4e09ce34 100644
--- a/morphlib/remoteartifactcache.py
+++ b/morphlib/remoteartifactcache.py
@@ -31,9 +31,9 @@ class GetError(cliapp.AppException):
 
     def __init__(self, cache, artifact):
         cliapp.AppException.__init__(
-            self, 'Failed to get the artifact %s with cache key %s '
+            self, 'Failed to get the artifact %s '
                   'from the artifact cache %s' %
-                  (artifact.basename(), artifact.cache_key, cache))
+                  (artifact.basename(), cache))
 
 
 class GetArtifactMetadataError(GetError):