Add 'build-mode' field for chunks in a stratum

Allowed values:

  staging: build with a staging chroot (default)
  test: build with the host's tools
  bootstrap: build with the host's tools, and do not include this
     chunk in the final stratum artifact

In the past, 'normal mode' has been used to describe building a chunk
with the host's tools. We don't want that mode to ever be used,
because it is a huge hole in reproducability, but we need to keep it
around to avoid making Morph's cmdtest suite depend on Baserock.
Hopefully naming it 'test' should discourage potential abusers.

It is unfortunate that the build tests now take a separate code path
compared to real-world usage of Morph. However, this is necessary to
avoid a circular dependency between Morph's test suite and the
build-essential stratum in Baserock.

We do whole-build testing of Baserock, too, so the 'staging' code path
is still tested outside of Morph. However, testing a staging area
requires populating it with at minimum a working shell, and this is a
bit too complex to go in Morph's test suite.

19 files changed, 84 insertions, 42 deletions

diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py
index 4b7956e0..76178b35 100644
--- a/morphlib/artifactresolver.py
+++ b/morphlib/artifactresolver.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012  Codethink Limited
+# Copyright (C) 2012-2013  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -220,6 +220,9 @@ class ArtifactResolver(object):
             for other_stratum in strata:
                 chunk_artifact.add_dependency(other_stratum)
 
+            # Resolve now to avoid a search for the parent morphology later
+            chunk_source.build_mode = info['build-mode']
+
             build_depends = info.get('build-depends', None)
 
             if build_depends is None:
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index ca097145..2de71d8f 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -226,13 +226,24 @@ class BuildCommand(object):
         self.get_sources(artifact)
         deps = self.get_recursive_deps(artifact)
         self.cache_artifacts_locally(deps)
-        staging_area = self.create_staging_area()
-        if artifact.source.morphology.needs_staging_area:
+
+        setup_mounts = False
+        if artifact.source.morphology['kind'] == 'chunk':
+            build_mode = artifact.source.build_mode
+
+            if build_mode not in ['bootstrap', 'staging', 'test']:
+                raise morphlib.Error(
+                    'Unknown build mode for chunk %s: %s' %
+                    (artifact.name, build_mode))
+
+            use_chroot = build_mode=='staging'
+            staging_area = self.create_staging_area(use_chroot)
             self.install_fillers(staging_area)
-            self.install_chunk_artifacts(staging_area, deps, artifact)
-            morphlib.builder2.ldconfig(self.app.runcmd,
-                                       staging_area.dirname)
-        self.build_and_cache(staging_area, artifact)
+            self.install_dependencies(staging_area, deps, artifact)
+        else:
+            staging_area = self.create_staging_area()
+
+        self.build_and_cache(staging_area, artifact, setup_mounts)
         self.remove_staging_area(staging_area)
 
     def get_recursive_deps(self, artifact):
@@ -301,13 +312,13 @@ class BuildCommand(object):
                     copy(self.rac.get_artifact_metadata(artifact, 'meta'),
                          self.lac.put_artifact_metadata(artifact, 'meta'))
 
-    def create_staging_area(self):
+    def create_staging_area(self, use_chroot=True):
         '''Create the staging area for building a single artifact.'''
 
         self.app.status(msg='Creating staging area')
         staging_dir = tempfile.mkdtemp(dir=self.app.settings['tempdir'])
         staging_area = morphlib.stagingarea.StagingArea(
-            self.app, staging_dir, self.build_env, False, {})
+            self.app, staging_dir, self.build_env, use_chroot, {})
         return staging_area
 
     def remove_staging_area(self, staging_area):
@@ -328,7 +339,7 @@ class BuildCommand(object):
                                 filename=filename)
                 staging_area.install_artifact(f)
 
-    def install_chunk_artifacts(self, staging_area, artifacts, parent_art):
+    def install_dependencies(self, staging_area, artifacts, target_artifact):
         '''Install chunk artifacts into staging area.
 
         We only ever care about chunk artifacts as build dependencies,
@@ -343,12 +354,15 @@ class BuildCommand(object):
             if artifact.source.morphology['kind'] != 'chunk':
                 continue
             self.app.status(msg='[%(name)s] Installing chunk %(chunk_name)s',
-                            name=parent_art.name,
+                            name=target_artifact.name,
                             chunk_name=artifact.name)
             handle = self.lac.get(artifact)
             staging_area.install_artifact(handle)
 
-    def build_and_cache(self, staging_area, artifact):
+        if target_artifact.source.build_mode == 'staging':
+            morphlib.builder2.ldconfig(self.app.runcmd, staging_area.dirname)
+
+    def build_and_cache(self, staging_area, artifact, setup_mounts):
         '''Build an artifact and put it into the local artifact cache.'''
 
         self.app.status(msg='Starting actual build: %(name)s',
@@ -356,5 +370,5 @@ class BuildCommand(object):
         setup_mounts = self.app.settings['staging-chroot']
         builder = morphlib.builder2.Builder(
             self.app, staging_area, self.lac, self.rac, self.lrc,
-            self.app.settings['max-jobs'], True)
+            self.app.settings['max-jobs'], setup_mounts)
         return builder.build_and_cache(artifact)
diff --git a/morphlib/buildenvironment.py b/morphlib/buildenvironment.py
index 29561220..6ba950ff 100644
--- a/morphlib/buildenvironment.py
+++ b/morphlib/buildenvironment.py
@@ -13,6 +13,7 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
+import copy
 import cliapp
 import os
 
@@ -21,6 +22,16 @@ import morphlib
 
 class BuildEnvironment():
 
+    '''Represents the build environment for an artifact
+
+       This should be as consistent as possible across builds, but some
+       artifacts will require tweaks. The intention of this object is
+       to create one once and call populate() to create an initial state
+       and when changes are required, call clone() to get another instance
+       which can be modified.
+
+    '''
+
     def __init__(self, settings, target, arch=None):
         '''Create a new BuildEnvironment object'''
 
@@ -88,7 +99,6 @@ class BuildEnvironment():
 
         if not settings['no-ccache']:
             self.extra_path.append(self._ccache_path)
-
 # FIXME: we should set CCACHE_BASEDIR so any objects that refer to their
 #        current directory get corrected. This improve the cache hit rate
 #            env['CCACHE_BASEDIR'] = self.tempdir.dirname
diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py
index d9ad5762..4573ad0d 100644
--- a/morphlib/cachekeycomputer.py
+++ b/morphlib/cachekeycomputer.py
@@ -87,6 +87,7 @@ class CacheKeyComputer(object):
 
         kind = artifact.source.morphology['kind']
         if kind == 'chunk':
+            keys['build-mode'] = artifact.source.build_mode
             keys['tree'] = artifact.source.tree
         elif kind in ('system', 'stratum'):
             morphology = artifact.source.morphology
diff --git a/morphlib/morph2.py b/morphlib/morph2.py
index 3cdf49a9..728fa533 100644
--- a/morphlib/morph2.py
+++ b/morphlib/morph2.py
@@ -52,7 +52,7 @@ class Morphology(object):
         'stratum': [
             ('chunks', []),
             ('description', ''),
-            ('build-depends', None)
+            ('build-depends', None),
         ],
         'system': [
             ('strata', []),
@@ -157,6 +157,8 @@ class Morphology(object):
                 self._set_default_value(source, 'morph', source['name'])
             if 'build-depends' not in source:
                 self._set_default_value(source, 'build-depends', None)
+            if 'build-mode' not in source:
+                self._set_default_value(source, 'build-mode', 'staging')
 
     def _parse_size(self, size):
         if isinstance(size, basestring):
diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py
index 76905eb9..7ae68697 100644
--- a/morphlib/morphologyfactory.py
+++ b/morphlib/morphologyfactory.py
@@ -113,7 +113,6 @@ class MorphologyFactory(object):
         name = morphology['name']
         morphology.builds_artifacts = [name + '-rootfs']
 
-        morphology.needs_staging_area = False
         morphology.needs_artifact_metadata_cached = False
 
     def _check_and_tweak_stratum(self, morphology, reponame, sha1, filename):
@@ -129,7 +128,6 @@ class MorphologyFactory(object):
                                      (filename, name))
 
         morphology.builds_artifacts = [morphology['name']]
-        morphology.needs_staging_area = False
         morphology.needs_artifact_metadata_cached = True
 
     def _check_and_tweak_chunk(self, morphology, reponame, sha1, filename):
@@ -140,5 +138,4 @@ class MorphologyFactory(object):
         else:
             morphology.builds_artifacts = [morphology['name']]
 
-        morphology.needs_staging_area = True
         morphology.needs_artifact_metadata_cached = False
diff --git a/morphlib/morphologyfactory_tests.py b/morphlib/morphologyfactory_tests.py
index 6e17df48..30cfb8fb 100644
--- a/morphlib/morphologyfactory_tests.py
+++ b/morphlib/morphologyfactory_tests.py
@@ -223,18 +223,6 @@ class MorphologyFactoryTests(unittest.TestCase):
         morph = self.mf.get_morphology('reponame', 'sha1', 'system.morph')
         self.assertEqual(morph.builds_artifacts, ['system-rootfs'])
 
-    def test_sets_needs_staging_for_chunk(self):
-        morph = self.mf.get_morphology('reponame', 'sha1', 'chunk.morph')
-        self.assertEqual(morph.needs_staging_area, True)
-
-    def test_does_not_set_needs_staging_for_stratum(self):
-        morph = self.mf.get_morphology('reponame', 'sha1', 'stratum.morph')
-        self.assertEqual(morph.needs_staging_area, False)
-
-    def test_does_not_set_needs_staging_for_system(self):
-        morph = self.mf.get_morphology('reponame', 'sha1', 'system.morph')
-        self.assertEqual(morph.needs_staging_area, False)
-
     def test_does_not_set_needs_artifact_metadata_cached_for_chunk(self):
         morph = self.mf.get_morphology('reponame', 'sha1', 'chunk.morph')
         self.assertEqual(morph.needs_artifact_metadata_cached, False)
diff --git a/morphlib/plugins/trebuchet_plugin.py b/morphlib/plugins/trebuchet_plugin.py
index 1ebffbf4..742d23c8 100644
--- a/morphlib/plugins/trebuchet_plugin.py
+++ b/morphlib/plugins/trebuchet_plugin.py
@@ -46,7 +46,8 @@ class TrebuchetPlugin(cliapp.Plugin):
         repo_name2, ref2, filename2 = args[4:7]
 
         app = self.app
-        build_env = morphlib.buildenvironment.BuildEnvironment(app.settings)
+        build_env = morphlib.buildenvironment.BuildEnvironment(
+            app.settings, morphlib.util.target(self.app.runcmd))
         ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env)
         lac, rac = morphlib.util.new_artifact_caches(app.settings)
         lrc, rrc = morphlib.util.new_repo_caches(app)
diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py
index ee3e444f..24b72867 100644
--- a/morphlib/stagingarea.py
+++ b/morphlib/stagingarea.py
@@ -37,7 +37,8 @@ class StagingArea(object):
 
     _base_path = ['/sbin', '/usr/sbin', '/bin', '/usr/bin']
 
-    def __init__(self, app, dirname, build_env, use_chroot=True, extra_env={}):
+    def __init__(self, app, dirname, build_env, use_chroot=True, extra_env={},
+                 extra_path=[]):
         self._app = app
         self.dirname = dirname
         self.builddirname = None
@@ -52,7 +53,8 @@ class StagingArea(object):
         if use_chroot:
             path = build_env.extra_path + self._base_path
         else:
-            full_path = [self.relative(p) for p in build_env.extra_path]
+            rel_path = build_env.extra_path
+            full_path = [os.path.normpath(dirname + p) for p in rel_path]
             path = full_path + os.environ['PATH'].split(':')
         self.env['PATH'] = ':'.join(path)
 
diff --git a/scripts/setup-3rd-party-strata b/scripts/setup-3rd-party-strata
index a2a8b1e5..d1cc320d 100644
--- a/scripts/setup-3rd-party-strata
+++ b/scripts/setup-3rd-party-strata
@@ -1,5 +1,5 @@
 #!/bin/sh
-# Copyright (C) 2012  Codethink Limited
+# Copyright (C) 2012-2013  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -55,6 +55,7 @@ cat <<EOF > "$1/$2.morph"
             "name": "hello",
             "repo": "test:$2-hello",
             "ref": "master",
+            "build-mode": "test",
             "build-depends": []
         }
     ]
diff --git a/tests.as-root/build-with-external-strata.script b/tests.as-root/build-with-external-strata.script
index 2d5d0fed..fd021399 100755
--- a/tests.as-root/build-with-external-strata.script
+++ b/tests.as-root/build-with-external-strata.script
@@ -38,6 +38,7 @@ cat <<EOF >> stratum2.morph
             "name": "linux",
             "repo": "test:kernel-repo",
             "ref": "master",
+            "build-mode": "test",
             "build-depends": []
         }
     ]
diff --git a/tests.as-root/setup b/tests.as-root/setup
index a85507e0..b9d5d477 100755
--- a/tests.as-root/setup
+++ b/tests.as-root/setup
@@ -110,6 +110,7 @@ chunks:
     - name: hello
       repo: test:chunk-repo
       ref: farrokh
+      build-mode: test
       build-depends: []
 EOF
 git add hello-stratum.morph
@@ -125,6 +126,7 @@ chunks:
     - name: tools
       repo: test:tools-repo
       ref: master
+      build-mode: test
       build-depends: []
 EOF
 git add tools-stratum.morph
@@ -153,6 +155,7 @@ chunks:
     - name: linux
       repo: test:kernel-repo
       ref: master
+      build-mode: test
       build-depends: []
 EOF
 git add linux-stratum.morph
diff --git a/tests.as-root/system-overlap.script b/tests.as-root/system-overlap.script
index cc308536..b8888491 100755
--- a/tests.as-root/system-overlap.script
+++ b/tests.as-root/system-overlap.script
@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# Copyright (C) 2011, 2012  Codethink Limited
+# Copyright (C) 2011-2013  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -57,12 +57,14 @@ cat <<EOF >foo-baz-stratum.morph
       "name": "overlap-foo-baz",
       "repo": "test:chunk-repo",
       "ref": "overlap",
+      "build-mode": "test",
       "build-depends": []
     },
     {
       "name": "linux",
       "repo": "test:kernel-repo",
       "ref": "master",
+      "build-mode": "test",
       "build-depends": ["overlap-foo-baz"]
     }
   ]
@@ -77,12 +79,14 @@ cat <<EOF >foo-barqux-stratum.morph
       "name": "overlap-foobar",
       "repo": "test:chunk-repo",
       "ref": "overlap",
+      "build-mode": "test",
       "build-depends": []
     },
     {
       "name": "overlap-fooqux",
       "repo": "test:chunk-repo",
       "ref": "overlap",
+      "build-mode": "test",
       "build-depends": ["overlap-foobar"]
     }
   ]
diff --git a/tests.as-root/tarball-image-is-sensible.setup b/tests.as-root/tarball-image-is-sensible.setup
index e159070c..fa904c2c 100755
--- a/tests.as-root/tarball-image-is-sensible.setup
+++ b/tests.as-root/tarball-image-is-sensible.setup
@@ -84,6 +84,7 @@ cat <<EOF > link-stratum.morph
             "name": "links",
             "repo": "test:chunk-repo",
             "ref": "tarball-links",
+            "build-mode": "test",
             "build-depends": []
         }
     ]
diff --git a/tests.branching/workflow-petrify.stdout b/tests.branching/workflow-petrify.stdout
index b572ed67..c86afa2e 100644
--- a/tests.branching/workflow-petrify.stdout
+++ b/tests.branching/workflow-petrify.stdout
@@ -31,6 +31,7 @@ test/petrify after petrifying:
             "name": "hello",
             "repo": "test:stratum2-hello",
             "ref": "f4730636e429149bb923fa16be3aa9802d484b23",
+            "build-mode": "test",
             "build-depends": [],
             "unpetrify-ref": "master"
         }
@@ -44,6 +45,7 @@ test/petrify after petrifying:
             "name": "hello",
             "repo": "test:stratum3-hello",
             "ref": "f4730636e429149bb923fa16be3aa9802d484b23",
+            "build-mode": "test",
             "build-depends": [],
             "unpetrify-ref": "master"
         }
@@ -83,6 +85,7 @@ test/petrify after editing a chunk:
             "name": "hello",
             "repo": "test:stratum2-hello",
             "ref": "test/petrify",
+            "build-mode": "test",
             "build-depends": []
         }
     ]
@@ -95,6 +98,7 @@ test/petrify after editing a chunk:
             "name": "hello",
             "repo": "test:stratum3-hello",
             "ref": "f4730636e429149bb923fa16be3aa9802d484b23",
+            "build-mode": "test",
             "build-depends": [],
             "unpetrify-ref": "master"
         }
@@ -134,6 +138,7 @@ test/unpetrify after unpetrifying:
             "name": "hello",
             "repo": "test:stratum2-hello",
             "ref": "test/petrify",
+            "build-mode": "test",
             "build-depends": []
         }
     ]
@@ -146,6 +151,7 @@ test/unpetrify after unpetrifying:
             "name": "hello",
             "repo": "test:stratum3-hello",
             "ref": "master",
+            "build-mode": "test",
             "build-depends": []
         }
     ]
diff --git a/tests.build/build-stratum-with-submodules.script b/tests.build/build-stratum-with-submodules.script
index f64ba9f6..c3c00578 100755
--- a/tests.build/build-stratum-with-submodules.script
+++ b/tests.build/build-stratum-with-submodules.script
@@ -55,7 +55,8 @@ cat <<EOF > "$morphs/hello-stratum.morph"
             "name": "parent",
             "repo": "test:parent-repo",
             "ref": "master",
-            "build-depends": []
+            "build-depends": [],
+            "build-mode": "test"
         }
     ]
 }
diff --git a/tests.build/setup b/tests.build/setup
index 935e388b..499dbb21 100755
--- a/tests.build/setup
+++ b/tests.build/setup
@@ -95,6 +95,7 @@ cat <<EOF > hello-stratum.morph
             "name": "hello",
             "repo": "test:chunk-repo",
             "ref": "farrokh",
+            "build-mode": "test",
             "build-depends": []
         }
     ]
diff --git a/tests.build/stratum-overlap-warns.setup b/tests.build/stratum-overlap-warns.setup
index 520a37a1..626f2094 100755
--- a/tests.build/stratum-overlap-warns.setup
+++ b/tests.build/stratum-overlap-warns.setup
@@ -34,25 +34,29 @@ cat <<EOF >hello-stratum.morph
       "name": "dirs",
       "repo": "test:chunk-repo",
       "ref": "overlap",
-      "build-depends": []
+      "build-depends": [],
+      "build-mode": "test"
     },
     {
       "name": "overlap-foobar",
       "repo": "test:chunk-repo",
       "ref": "overlap",
-      "build-depends": ["dirs"]
+      "build-depends": ["dirs"],
+      "build-mode": "test"
     },
     {
       "name": "overlap-fooqux",
       "repo": "test:chunk-repo",
       "ref": "overlap",
-      "build-depends": ["overlap-foobar"]
+      "build-depends": ["overlap-foobar"],
+      "build-mode": "test"
     },
     {
       "name": "overlap-foo-baz",
       "repo": "test:chunk-repo",
       "ref": "overlap",
-      "build-depends": ["overlap-fooqux"]
+      "build-depends": ["overlap-fooqux"],
+      "build-mode": "test"
     }
   ]
 }
diff --git a/tests.deploy/setup b/tests.deploy/setup
index b0054fa5..584ce039 100755
--- a/tests.deploy/setup
+++ b/tests.deploy/setup
@@ -100,7 +100,8 @@ cat <<EOF > hello-stratum.morph
             "name": "hello",
             "repo": "test:chunk-repo",
             "ref": "farrokh",
-            "build-depends": []
+            "build-depends": [],
+            "build-mode": "test"
         }
     ]
 }
@@ -140,7 +141,8 @@ cat <<EOF > linux-stratum.morph
             "name": "linux",
             "repo": "test:kernel-repo",
             "ref": "master",
-            "build-depends": []
+            "build-depends": [],
+            "build-mode": "test"
         }
     ]
 }