diff options
author | Sam Thursfield <sam.thursfield@codethink.co.uk> | 2013-02-06 10:23:57 +0000 |
---|---|---|
committer | Sam Thursfield <sam.thursfield@codethink.co.uk> | 2013-03-13 15:20:02 +0000 |
commit | 84807d4d7c23f45d4f0a0f87e6c7ba7ba7470936 (patch) | |
tree | 7a0a2c96fd33b471974d437ac427fc98f5e9c38c | |
parent | 92d98f57eaa6e0871b70d0d0ab1db879cf3ea47a (diff) | |
download | morph-84807d4d7c23f45d4f0a0f87e6c7ba7ba7470936.tar.gz |
Add 'build-mode' field for chunks in a stratum
Allowed values:
staging: build with a staging chroot (default)
test: build with the host's tools
bootstrap: build with the host's tools, and do not include this
chunk in the final stratum artifact
In the past, 'normal mode' has been used to describe building a chunk
with the host's tools. We don't want that mode to ever be used,
because it is a huge hole in reproducability, but we need to keep it
around to avoid making Morph's cmdtest suite depend on Baserock.
Hopefully naming it 'test' should discourage potential abusers.
It is unfortunate that the build tests now take a separate code path
compared to real-world usage of Morph. However, this is necessary to
avoid a circular dependency between Morph's test suite and the
build-essential stratum in Baserock.
We do whole-build testing of Baserock, too, so the 'staging' code path
is still tested outside of Morph. However, testing a staging area
requires populating it with at minimum a working shell, and this is a
bit too complex to go in Morph's test suite.
-rw-r--r-- | morphlib/artifactresolver.py | 5 | ||||
-rw-r--r-- | morphlib/buildcommand.py | 38 | ||||
-rw-r--r-- | morphlib/buildenvironment.py | 12 | ||||
-rw-r--r-- | morphlib/cachekeycomputer.py | 1 | ||||
-rw-r--r-- | morphlib/morph2.py | 4 | ||||
-rw-r--r-- | morphlib/morphologyfactory.py | 3 | ||||
-rw-r--r-- | morphlib/morphologyfactory_tests.py | 12 | ||||
-rw-r--r-- | morphlib/plugins/trebuchet_plugin.py | 3 | ||||
-rw-r--r-- | morphlib/stagingarea.py | 6 | ||||
-rw-r--r-- | scripts/setup-3rd-party-strata | 3 | ||||
-rwxr-xr-x | tests.as-root/build-with-external-strata.script | 1 | ||||
-rwxr-xr-x | tests.as-root/setup | 3 | ||||
-rwxr-xr-x | tests.as-root/system-overlap.script | 6 | ||||
-rwxr-xr-x | tests.as-root/tarball-image-is-sensible.setup | 1 | ||||
-rw-r--r-- | tests.branching/workflow-petrify.stdout | 6 | ||||
-rwxr-xr-x | tests.build/build-stratum-with-submodules.script | 3 | ||||
-rwxr-xr-x | tests.build/setup | 1 | ||||
-rwxr-xr-x | tests.build/stratum-overlap-warns.setup | 12 | ||||
-rwxr-xr-x | tests.deploy/setup | 6 |
19 files changed, 84 insertions, 42 deletions
diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index 4b7956e0..76178b35 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -220,6 +220,9 @@ class ArtifactResolver(object): for other_stratum in strata: chunk_artifact.add_dependency(other_stratum) + # Resolve now to avoid a search for the parent morphology later + chunk_source.build_mode = info['build-mode'] + build_depends = info.get('build-depends', None) if build_depends is None: diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index ca097145..2de71d8f 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -226,13 +226,24 @@ class BuildCommand(object): self.get_sources(artifact) deps = self.get_recursive_deps(artifact) self.cache_artifacts_locally(deps) - staging_area = self.create_staging_area() - if artifact.source.morphology.needs_staging_area: + + setup_mounts = False + if artifact.source.morphology['kind'] == 'chunk': + build_mode = artifact.source.build_mode + + if build_mode not in ['bootstrap', 'staging', 'test']: + raise morphlib.Error( + 'Unknown build mode for chunk %s: %s' % + (artifact.name, build_mode)) + + use_chroot = build_mode=='staging' + staging_area = self.create_staging_area(use_chroot) self.install_fillers(staging_area) - self.install_chunk_artifacts(staging_area, deps, artifact) - morphlib.builder2.ldconfig(self.app.runcmd, - staging_area.dirname) - self.build_and_cache(staging_area, artifact) + self.install_dependencies(staging_area, deps, artifact) + else: + staging_area = self.create_staging_area() + + self.build_and_cache(staging_area, artifact, setup_mounts) self.remove_staging_area(staging_area) def get_recursive_deps(self, artifact): @@ -301,13 +312,13 @@ class BuildCommand(object): copy(self.rac.get_artifact_metadata(artifact, 'meta'), self.lac.put_artifact_metadata(artifact, 'meta')) - def create_staging_area(self): + def create_staging_area(self, use_chroot=True): '''Create the staging area for building a single artifact.''' self.app.status(msg='Creating staging area') staging_dir = tempfile.mkdtemp(dir=self.app.settings['tempdir']) staging_area = morphlib.stagingarea.StagingArea( - self.app, staging_dir, self.build_env, False, {}) + self.app, staging_dir, self.build_env, use_chroot, {}) return staging_area def remove_staging_area(self, staging_area): @@ -328,7 +339,7 @@ class BuildCommand(object): filename=filename) staging_area.install_artifact(f) - def install_chunk_artifacts(self, staging_area, artifacts, parent_art): + def install_dependencies(self, staging_area, artifacts, target_artifact): '''Install chunk artifacts into staging area. We only ever care about chunk artifacts as build dependencies, @@ -343,12 +354,15 @@ class BuildCommand(object): if artifact.source.morphology['kind'] != 'chunk': continue self.app.status(msg='[%(name)s] Installing chunk %(chunk_name)s', - name=parent_art.name, + name=target_artifact.name, chunk_name=artifact.name) handle = self.lac.get(artifact) staging_area.install_artifact(handle) - def build_and_cache(self, staging_area, artifact): + if target_artifact.source.build_mode == 'staging': + morphlib.builder2.ldconfig(self.app.runcmd, staging_area.dirname) + + def build_and_cache(self, staging_area, artifact, setup_mounts): '''Build an artifact and put it into the local artifact cache.''' self.app.status(msg='Starting actual build: %(name)s', @@ -356,5 +370,5 @@ class BuildCommand(object): setup_mounts = self.app.settings['staging-chroot'] builder = morphlib.builder2.Builder( self.app, staging_area, self.lac, self.rac, self.lrc, - self.app.settings['max-jobs'], True) + self.app.settings['max-jobs'], setup_mounts) return builder.build_and_cache(artifact) diff --git a/morphlib/buildenvironment.py b/morphlib/buildenvironment.py index 29561220..6ba950ff 100644 --- a/morphlib/buildenvironment.py +++ b/morphlib/buildenvironment.py @@ -13,6 +13,7 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import copy import cliapp import os @@ -21,6 +22,16 @@ import morphlib class BuildEnvironment(): + '''Represents the build environment for an artifact + + This should be as consistent as possible across builds, but some + artifacts will require tweaks. The intention of this object is + to create one once and call populate() to create an initial state + and when changes are required, call clone() to get another instance + which can be modified. + + ''' + def __init__(self, settings, target, arch=None): '''Create a new BuildEnvironment object''' @@ -88,7 +99,6 @@ class BuildEnvironment(): if not settings['no-ccache']: self.extra_path.append(self._ccache_path) - # FIXME: we should set CCACHE_BASEDIR so any objects that refer to their # current directory get corrected. This improve the cache hit rate # env['CCACHE_BASEDIR'] = self.tempdir.dirname diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index d9ad5762..4573ad0d 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -87,6 +87,7 @@ class CacheKeyComputer(object): kind = artifact.source.morphology['kind'] if kind == 'chunk': + keys['build-mode'] = artifact.source.build_mode keys['tree'] = artifact.source.tree elif kind in ('system', 'stratum'): morphology = artifact.source.morphology diff --git a/morphlib/morph2.py b/morphlib/morph2.py index 3cdf49a9..728fa533 100644 --- a/morphlib/morph2.py +++ b/morphlib/morph2.py @@ -52,7 +52,7 @@ class Morphology(object): 'stratum': [ ('chunks', []), ('description', ''), - ('build-depends', None) + ('build-depends', None), ], 'system': [ ('strata', []), @@ -157,6 +157,8 @@ class Morphology(object): self._set_default_value(source, 'morph', source['name']) if 'build-depends' not in source: self._set_default_value(source, 'build-depends', None) + if 'build-mode' not in source: + self._set_default_value(source, 'build-mode', 'staging') def _parse_size(self, size): if isinstance(size, basestring): diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py index 76905eb9..7ae68697 100644 --- a/morphlib/morphologyfactory.py +++ b/morphlib/morphologyfactory.py @@ -113,7 +113,6 @@ class MorphologyFactory(object): name = morphology['name'] morphology.builds_artifacts = [name + '-rootfs'] - morphology.needs_staging_area = False morphology.needs_artifact_metadata_cached = False def _check_and_tweak_stratum(self, morphology, reponame, sha1, filename): @@ -129,7 +128,6 @@ class MorphologyFactory(object): (filename, name)) morphology.builds_artifacts = [morphology['name']] - morphology.needs_staging_area = False morphology.needs_artifact_metadata_cached = True def _check_and_tweak_chunk(self, morphology, reponame, sha1, filename): @@ -140,5 +138,4 @@ class MorphologyFactory(object): else: morphology.builds_artifacts = [morphology['name']] - morphology.needs_staging_area = True morphology.needs_artifact_metadata_cached = False diff --git a/morphlib/morphologyfactory_tests.py b/morphlib/morphologyfactory_tests.py index 6e17df48..30cfb8fb 100644 --- a/morphlib/morphologyfactory_tests.py +++ b/morphlib/morphologyfactory_tests.py @@ -223,18 +223,6 @@ class MorphologyFactoryTests(unittest.TestCase): morph = self.mf.get_morphology('reponame', 'sha1', 'system.morph') self.assertEqual(morph.builds_artifacts, ['system-rootfs']) - def test_sets_needs_staging_for_chunk(self): - morph = self.mf.get_morphology('reponame', 'sha1', 'chunk.morph') - self.assertEqual(morph.needs_staging_area, True) - - def test_does_not_set_needs_staging_for_stratum(self): - morph = self.mf.get_morphology('reponame', 'sha1', 'stratum.morph') - self.assertEqual(morph.needs_staging_area, False) - - def test_does_not_set_needs_staging_for_system(self): - morph = self.mf.get_morphology('reponame', 'sha1', 'system.morph') - self.assertEqual(morph.needs_staging_area, False) - def test_does_not_set_needs_artifact_metadata_cached_for_chunk(self): morph = self.mf.get_morphology('reponame', 'sha1', 'chunk.morph') self.assertEqual(morph.needs_artifact_metadata_cached, False) diff --git a/morphlib/plugins/trebuchet_plugin.py b/morphlib/plugins/trebuchet_plugin.py index 1ebffbf4..742d23c8 100644 --- a/morphlib/plugins/trebuchet_plugin.py +++ b/morphlib/plugins/trebuchet_plugin.py @@ -46,7 +46,8 @@ class TrebuchetPlugin(cliapp.Plugin): repo_name2, ref2, filename2 = args[4:7] app = self.app - build_env = morphlib.buildenvironment.BuildEnvironment(app.settings) + build_env = morphlib.buildenvironment.BuildEnvironment( + app.settings, morphlib.util.target(self.app.runcmd)) ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) lac, rac = morphlib.util.new_artifact_caches(app.settings) lrc, rrc = morphlib.util.new_repo_caches(app) diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index ee3e444f..24b72867 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -37,7 +37,8 @@ class StagingArea(object): _base_path = ['/sbin', '/usr/sbin', '/bin', '/usr/bin'] - def __init__(self, app, dirname, build_env, use_chroot=True, extra_env={}): + def __init__(self, app, dirname, build_env, use_chroot=True, extra_env={}, + extra_path=[]): self._app = app self.dirname = dirname self.builddirname = None @@ -52,7 +53,8 @@ class StagingArea(object): if use_chroot: path = build_env.extra_path + self._base_path else: - full_path = [self.relative(p) for p in build_env.extra_path] + rel_path = build_env.extra_path + full_path = [os.path.normpath(dirname + p) for p in rel_path] path = full_path + os.environ['PATH'].split(':') self.env['PATH'] = ':'.join(path) diff --git a/scripts/setup-3rd-party-strata b/scripts/setup-3rd-party-strata index a2a8b1e5..d1cc320d 100644 --- a/scripts/setup-3rd-party-strata +++ b/scripts/setup-3rd-party-strata @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -55,6 +55,7 @@ cat <<EOF > "$1/$2.morph" "name": "hello", "repo": "test:$2-hello", "ref": "master", + "build-mode": "test", "build-depends": [] } ] diff --git a/tests.as-root/build-with-external-strata.script b/tests.as-root/build-with-external-strata.script index 2d5d0fed..fd021399 100755 --- a/tests.as-root/build-with-external-strata.script +++ b/tests.as-root/build-with-external-strata.script @@ -38,6 +38,7 @@ cat <<EOF >> stratum2.morph "name": "linux", "repo": "test:kernel-repo", "ref": "master", + "build-mode": "test", "build-depends": [] } ] diff --git a/tests.as-root/setup b/tests.as-root/setup index a85507e0..b9d5d477 100755 --- a/tests.as-root/setup +++ b/tests.as-root/setup @@ -110,6 +110,7 @@ chunks: - name: hello repo: test:chunk-repo ref: farrokh + build-mode: test build-depends: [] EOF git add hello-stratum.morph @@ -125,6 +126,7 @@ chunks: - name: tools repo: test:tools-repo ref: master + build-mode: test build-depends: [] EOF git add tools-stratum.morph @@ -153,6 +155,7 @@ chunks: - name: linux repo: test:kernel-repo ref: master + build-mode: test build-depends: [] EOF git add linux-stratum.morph diff --git a/tests.as-root/system-overlap.script b/tests.as-root/system-overlap.script index cc308536..b8888491 100755 --- a/tests.as-root/system-overlap.script +++ b/tests.as-root/system-overlap.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011, 2012 Codethink Limited +# Copyright (C) 2011-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -57,12 +57,14 @@ cat <<EOF >foo-baz-stratum.morph "name": "overlap-foo-baz", "repo": "test:chunk-repo", "ref": "overlap", + "build-mode": "test", "build-depends": [] }, { "name": "linux", "repo": "test:kernel-repo", "ref": "master", + "build-mode": "test", "build-depends": ["overlap-foo-baz"] } ] @@ -77,12 +79,14 @@ cat <<EOF >foo-barqux-stratum.morph "name": "overlap-foobar", "repo": "test:chunk-repo", "ref": "overlap", + "build-mode": "test", "build-depends": [] }, { "name": "overlap-fooqux", "repo": "test:chunk-repo", "ref": "overlap", + "build-mode": "test", "build-depends": ["overlap-foobar"] } ] diff --git a/tests.as-root/tarball-image-is-sensible.setup b/tests.as-root/tarball-image-is-sensible.setup index e159070c..fa904c2c 100755 --- a/tests.as-root/tarball-image-is-sensible.setup +++ b/tests.as-root/tarball-image-is-sensible.setup @@ -84,6 +84,7 @@ cat <<EOF > link-stratum.morph "name": "links", "repo": "test:chunk-repo", "ref": "tarball-links", + "build-mode": "test", "build-depends": [] } ] diff --git a/tests.branching/workflow-petrify.stdout b/tests.branching/workflow-petrify.stdout index b572ed67..c86afa2e 100644 --- a/tests.branching/workflow-petrify.stdout +++ b/tests.branching/workflow-petrify.stdout @@ -31,6 +31,7 @@ test/petrify after petrifying: "name": "hello", "repo": "test:stratum2-hello", "ref": "f4730636e429149bb923fa16be3aa9802d484b23", + "build-mode": "test", "build-depends": [], "unpetrify-ref": "master" } @@ -44,6 +45,7 @@ test/petrify after petrifying: "name": "hello", "repo": "test:stratum3-hello", "ref": "f4730636e429149bb923fa16be3aa9802d484b23", + "build-mode": "test", "build-depends": [], "unpetrify-ref": "master" } @@ -83,6 +85,7 @@ test/petrify after editing a chunk: "name": "hello", "repo": "test:stratum2-hello", "ref": "test/petrify", + "build-mode": "test", "build-depends": [] } ] @@ -95,6 +98,7 @@ test/petrify after editing a chunk: "name": "hello", "repo": "test:stratum3-hello", "ref": "f4730636e429149bb923fa16be3aa9802d484b23", + "build-mode": "test", "build-depends": [], "unpetrify-ref": "master" } @@ -134,6 +138,7 @@ test/unpetrify after unpetrifying: "name": "hello", "repo": "test:stratum2-hello", "ref": "test/petrify", + "build-mode": "test", "build-depends": [] } ] @@ -146,6 +151,7 @@ test/unpetrify after unpetrifying: "name": "hello", "repo": "test:stratum3-hello", "ref": "master", + "build-mode": "test", "build-depends": [] } ] diff --git a/tests.build/build-stratum-with-submodules.script b/tests.build/build-stratum-with-submodules.script index f64ba9f6..c3c00578 100755 --- a/tests.build/build-stratum-with-submodules.script +++ b/tests.build/build-stratum-with-submodules.script @@ -55,7 +55,8 @@ cat <<EOF > "$morphs/hello-stratum.morph" "name": "parent", "repo": "test:parent-repo", "ref": "master", - "build-depends": [] + "build-depends": [], + "build-mode": "test" } ] } diff --git a/tests.build/setup b/tests.build/setup index 935e388b..499dbb21 100755 --- a/tests.build/setup +++ b/tests.build/setup @@ -95,6 +95,7 @@ cat <<EOF > hello-stratum.morph "name": "hello", "repo": "test:chunk-repo", "ref": "farrokh", + "build-mode": "test", "build-depends": [] } ] diff --git a/tests.build/stratum-overlap-warns.setup b/tests.build/stratum-overlap-warns.setup index 520a37a1..626f2094 100755 --- a/tests.build/stratum-overlap-warns.setup +++ b/tests.build/stratum-overlap-warns.setup @@ -34,25 +34,29 @@ cat <<EOF >hello-stratum.morph "name": "dirs", "repo": "test:chunk-repo", "ref": "overlap", - "build-depends": [] + "build-depends": [], + "build-mode": "test" }, { "name": "overlap-foobar", "repo": "test:chunk-repo", "ref": "overlap", - "build-depends": ["dirs"] + "build-depends": ["dirs"], + "build-mode": "test" }, { "name": "overlap-fooqux", "repo": "test:chunk-repo", "ref": "overlap", - "build-depends": ["overlap-foobar"] + "build-depends": ["overlap-foobar"], + "build-mode": "test" }, { "name": "overlap-foo-baz", "repo": "test:chunk-repo", "ref": "overlap", - "build-depends": ["overlap-fooqux"] + "build-depends": ["overlap-fooqux"], + "build-mode": "test" } ] } diff --git a/tests.deploy/setup b/tests.deploy/setup index b0054fa5..584ce039 100755 --- a/tests.deploy/setup +++ b/tests.deploy/setup @@ -100,7 +100,8 @@ cat <<EOF > hello-stratum.morph "name": "hello", "repo": "test:chunk-repo", "ref": "farrokh", - "build-depends": [] + "build-depends": [], + "build-mode": "test" } ] } @@ -140,7 +141,8 @@ cat <<EOF > linux-stratum.morph "name": "linux", "repo": "test:kernel-repo", "ref": "master", - "build-depends": [] + "build-depends": [], + "build-mode": "test" } ] } |