Refactor build process

Reorganise the build_artifact() and build_artifacts() functions to
allow more complex work when setting up chunk builds in
build_artifact().

The staging area now holds the BuildEnvironment object (the
environment variables that should be set during build). This makes sense
because all build commands should be run inside the staging area and
therefore through the StagingArea object.

The BuildEnvironment object is now considered immutable after it is
created.  The environment is used in cache key computation when
computing what artifacts are required; if it changes after that point
we risk either computing different artifact keys for the same artifact
or missing data in the cache key that should be included in the hash.
Better to force changes into a separate 'extra_env' variable.

1 files changed, 40 insertions, 52 deletions

diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index c7c650d3..56c7fae8 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -181,19 +181,38 @@ class BuildCommand(object):
         assert len(maybe) == 1
         return maybe.pop()
 
-    def build_in_order(self, artifact):
+    def build_in_order(self, root_artifact):
         '''Build everything specified in a build order.'''
-        self.app.status(msg='Building according to build ordering',
-                        chatty=True)
 
-        artifacts = artifact.walk()
+        self.app.status(msg='Building a set of artifacts', chatty=True)
+        artifacts = root_artifact.walk()
         old_prefix = self.app.status_prefix
         for i, a in enumerate(artifacts):
             self.app.status_prefix = (
                 old_prefix + '[Build %d/%d] ' % ((i+1), len(artifacts)))
-            self.build_artifact(a)
+
+            self.app.status(msg='Checking if %(kind)s %(name)s needs building',
+                            kind=a.source.morphology['kind'], name=a.name)
+
+            if self.is_built(a):
+                self.app.status(msg='The %(kind)s %(name)s is already built',
+                                kind=a.source.morphology['kind'], name=a.name)
+                self.cache_artifacts_locally([a])
+            else:
+                self.app.status(msg='Building %(kind)s %(name)s',
+                                kind=a.source.morphology['kind'], name=a.name)
+                self.build_artifact(a)
+
+            self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s',
+                            kind=a.source.morphology['kind'], name=a.name,
+                            cachepath=self.lac.artifact_filename(a),
+                            chatty=(a.source.morphology['kind'] != "system"))
         self.app.status_prefix = old_prefix
 
+    def is_built(self, artifact):
+        '''Does either cache already have the artifact?'''
+        return self.lac.has(artifact) or (self.rac and self.rac.has(artifact))
+
     def build_artifact(self, artifact):
         '''Build one artifact.
 
@@ -201,52 +220,20 @@ class BuildCommand(object):
         in either the local or remote cache already.
 
         '''
-
-        self.app.status(msg='Checking if %(kind)s %(name)s needs building',
-                        kind=artifact.source.morphology['kind'],
-                        name=artifact.name)
-
-        if self.is_built(artifact):
-            self.app.status(msg='The %(kind)s %(name)s is already built',
-                            kind=artifact.source.morphology['kind'],
-                            name=artifact.name)
-            self.cache_artifacts_locally([artifact])
-        else:
-            self.app.status(msg='Building %(kind)s %(name)s',
-                            kind=artifact.source.morphology['kind'],
-                            name=artifact.name)
-            self.get_sources(artifact)
-            deps = self.get_recursive_deps(artifact)
-            self.cache_artifacts_locally(deps)
-            staging_area = self.create_staging_area(artifact)
-            if artifact.source.morphology.needs_staging_area:
-                self.install_fillers(staging_area)
-                self.install_chunk_artifacts(staging_area,
-                                             deps, artifact)
-                morphlib.builder2.ldconfig(self.app.runcmd,
-                                           staging_area.dirname)
-            self.build_and_cache(staging_area, artifact)
-            self.remove_staging_area(staging_area)
-        self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s',
-                        kind=artifact.source.morphology['kind'],
-                        name=artifact.name,
-                        cachepath=self.lac.artifact_filename(artifact),
-                        chatty=(artifact.source.morphology['kind'] !=
-                                "system"))
-
-    def is_built(self, artifact):
-        '''Does either cache already have the artifact?'''
-        return self.lac.has(artifact) or (self.rac and self.rac.has(artifact))
+        self.get_sources(artifact)
+        deps = self.get_recursive_deps(artifact)
+        self.cache_artifacts_locally(deps)
+        staging_area = self.create_staging_area()
+        if artifact.source.morphology.needs_staging_area:
+            self.install_fillers(staging_area)
+            self.install_chunk_artifacts(staging_area, deps, artifact)
+            morphlib.builder2.ldconfig(self.app.runcmd,
+                                       staging_area.dirname)
+        self.build_and_cache(staging_area, artifact)
+        self.remove_staging_area(staging_area)
 
     def get_recursive_deps(self, artifact):
-        done = set()
-        todo = set((artifact,))
-        while todo:
-            for a in todo.pop().dependencies:
-                if a not in done:
-                    done.add(a)
-                    todo.add(a)
-        return done
+        return artifact.walk()[:-1]
 
     def get_sources(self, artifact):
         '''Update the local git repository cache with the sources.'''
@@ -311,12 +298,13 @@ class BuildCommand(object):
                     copy(self.rac.get_artifact_metadata(artifact, 'meta'),
                          self.lac.put_artifact_metadata(artifact, 'meta'))
 
-    def create_staging_area(self, artifact):
+    def create_staging_area(self):
         '''Create the staging area for building a single artifact.'''
 
         self.app.status(msg='Creating staging area')
         staging_dir = tempfile.mkdtemp(dir=self.app.settings['tempdir'])
-        staging_area = morphlib.stagingarea.StagingArea(self.app, staging_dir)
+        staging_area = morphlib.stagingarea.StagingArea(
+            self.app, staging_dir, self.build_env, False, {})
         return staging_area
 
     def remove_staging_area(self, staging_area):
@@ -365,5 +353,5 @@ class BuildCommand(object):
         setup_mounts = self.app.settings['staging-chroot']
         builder = morphlib.builder2.Builder(
             self.app, staging_area, self.lac, self.rac, self.lrc,
-            self.build_env, self.app.settings['max-jobs'], True)
+            self.app.settings['max-jobs'], True)
         return builder.build_and_cache(artifact)