diff options
| -rw-r--r-- | morphlib/artifactresolver.py | 5 | ||||
| -rw-r--r-- | morphlib/buildcommand.py | 38 | ||||
| -rw-r--r-- | morphlib/buildenvironment.py | 12 | ||||
| -rw-r--r-- | morphlib/cachekeycomputer.py | 1 | ||||
| -rw-r--r-- | morphlib/morph2.py | 4 | ||||
| -rw-r--r-- | morphlib/morphologyfactory.py | 3 | ||||
| -rw-r--r-- | morphlib/morphologyfactory_tests.py | 12 | ||||
| -rw-r--r-- | morphlib/plugins/trebuchet_plugin.py | 3 | ||||
| -rw-r--r-- | morphlib/stagingarea.py | 6 | ||||
| -rw-r--r-- | scripts/setup-3rd-party-strata | 3 | ||||
| -rwxr-xr-x | tests.as-root/build-with-external-strata.script | 1 | ||||
| -rwxr-xr-x | tests.as-root/setup | 3 | ||||
| -rwxr-xr-x | tests.as-root/system-overlap.script | 6 | ||||
| -rwxr-xr-x | tests.as-root/tarball-image-is-sensible.setup | 1 | ||||
| -rw-r--r-- | tests.branching/workflow-petrify.stdout | 6 | ||||
| -rwxr-xr-x | tests.build/build-stratum-with-submodules.script | 3 | ||||
| -rwxr-xr-x | tests.build/setup | 1 | ||||
| -rwxr-xr-x | tests.build/stratum-overlap-warns.setup | 12 | ||||
| -rwxr-xr-x | tests.deploy/setup | 6 |
19 files changed, 84 insertions, 42 deletions
diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index 4b7956e0..76178b35 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -220,6 +220,9 @@ class ArtifactResolver(object): for other_stratum in strata: chunk_artifact.add_dependency(other_stratum) + # Resolve now to avoid a search for the parent morphology later + chunk_source.build_mode = info['build-mode'] + build_depends = info.get('build-depends', None) if build_depends is None: diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index ca097145..2de71d8f 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -226,13 +226,24 @@ class BuildCommand(object): self.get_sources(artifact) deps = self.get_recursive_deps(artifact) self.cache_artifacts_locally(deps) - staging_area = self.create_staging_area() - if artifact.source.morphology.needs_staging_area: + + setup_mounts = False + if artifact.source.morphology['kind'] == 'chunk': + build_mode = artifact.source.build_mode + + if build_mode not in ['bootstrap', 'staging', 'test']: + raise morphlib.Error( + 'Unknown build mode for chunk %s: %s' % + (artifact.name, build_mode)) + + use_chroot = build_mode=='staging' + staging_area = self.create_staging_area(use_chroot) self.install_fillers(staging_area) - self.install_chunk_artifacts(staging_area, deps, artifact) - morphlib.builder2.ldconfig(self.app.runcmd, - staging_area.dirname) - self.build_and_cache(staging_area, artifact) + self.install_dependencies(staging_area, deps, artifact) + else: + staging_area = self.create_staging_area() + + self.build_and_cache(staging_area, artifact, setup_mounts) self.remove_staging_area(staging_area) def get_recursive_deps(self, artifact): @@ -301,13 +312,13 @@ class BuildCommand(object): copy(self.rac.get_artifact_metadata(artifact, 'meta'), self.lac.put_artifact_metadata(artifact, 'meta')) - def create_staging_area(self): + def create_staging_area(self, use_chroot=True): '''Create the staging area for building a single artifact.''' self.app.status(msg='Creating staging area') staging_dir = tempfile.mkdtemp(dir=self.app.settings['tempdir']) staging_area = morphlib.stagingarea.StagingArea( - self.app, staging_dir, self.build_env, False, {}) + self.app, staging_dir, self.build_env, use_chroot, {}) return staging_area def remove_staging_area(self, staging_area): @@ -328,7 +339,7 @@ class BuildCommand(object): filename=filename) staging_area.install_artifact(f) - def install_chunk_artifacts(self, staging_area, artifacts, parent_art): + def install_dependencies(self, staging_area, artifacts, target_artifact): '''Install chunk artifacts into staging area. We only ever care about chunk artifacts as build dependencies, @@ -343,12 +354,15 @@ class BuildCommand(object): if artifact.source.morphology['kind'] != 'chunk': continue self.app.status(msg='[%(name)s] Installing chunk %(chunk_name)s', - name=parent_art.name, + name=target_artifact.name, chunk_name=artifact.name) handle = self.lac.get(artifact) staging_area.install_artifact(handle) - def build_and_cache(self, staging_area, artifact): + if target_artifact.source.build_mode == 'staging': + morphlib.builder2.ldconfig(self.app.runcmd, staging_area.dirname) + + def build_and_cache(self, staging_area, artifact, setup_mounts): '''Build an artifact and put it into the local artifact cache.''' self.app.status(msg='Starting actual build: %(name)s', @@ -356,5 +370,5 @@ class BuildCommand(object): setup_mounts = self.app.settings['staging-chroot'] builder = morphlib.builder2.Builder( self.app, staging_area, self.lac, self.rac, self.lrc, - self.app.settings['max-jobs'], True) + self.app.settings['max-jobs'], setup_mounts) return builder.build_and_cache(artifact) diff --git a/morphlib/buildenvironment.py b/morphlib/buildenvironment.py index 29561220..6ba950ff 100644 --- a/morphlib/buildenvironment.py +++ b/morphlib/buildenvironment.py @@ -13,6 +13,7 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import copy import cliapp import os @@ -21,6 +22,16 @@ import morphlib class BuildEnvironment(): + '''Represents the build environment for an artifact + + This should be as consistent as possible across builds, but some + artifacts will require tweaks. The intention of this object is + to create one once and call populate() to create an initial state + and when changes are required, call clone() to get another instance + which can be modified. + + ''' + def __init__(self, settings, target, arch=None): '''Create a new BuildEnvironment object''' @@ -88,7 +99,6 @@ class BuildEnvironment(): if not settings['no-ccache']: self.extra_path.append(self._ccache_path) - # FIXME: we should set CCACHE_BASEDIR so any objects that refer to their # current directory get corrected. This improve the cache hit rate # env['CCACHE_BASEDIR'] = self.tempdir.dirname diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index d9ad5762..4573ad0d 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -87,6 +87,7 @@ class CacheKeyComputer(object): kind = artifact.source.morphology['kind'] if kind == 'chunk': + keys['build-mode'] = artifact.source.build_mode keys['tree'] = artifact.source.tree elif kind in ('system', 'stratum'): morphology = artifact.source.morphology diff --git a/morphlib/morph2.py b/morphlib/morph2.py index 3cdf49a9..728fa533 100644 --- a/morphlib/morph2.py +++ b/morphlib/morph2.py @@ -52,7 +52,7 @@ class Morphology(object): 'stratum': [ ('chunks', []), ('description', ''), - ('build-depends', None) + ('build-depends', None), ], 'system': [ ('strata', []), @@ -157,6 +157,8 @@ class Morphology(object): self._set_default_value(source, 'morph', source['name']) if 'build-depends' not in source: self._set_default_value(source, 'build-depends', None) + if 'build-mode' not in source: + self._set_default_value(source, 'build-mode', 'staging') def _parse_size(self, size): if isinstance(size, basestring): diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py index 76905eb9..7ae68697 100644 --- a/morphlib/morphologyfactory.py +++ b/morphlib/morphologyfactory.py @@ -113,7 +113,6 @@ class MorphologyFactory(object): name = morphology['name'] morphology.builds_artifacts = [name + '-rootfs'] - morphology.needs_staging_area = False morphology.needs_artifact_metadata_cached = False def _check_and_tweak_stratum(self, morphology, reponame, sha1, filename): @@ -129,7 +128,6 @@ class MorphologyFactory(object): (filename, name)) morphology.builds_artifacts = [morphology['name']] - morphology.needs_staging_area = False morphology.needs_artifact_metadata_cached = True def _check_and_tweak_chunk(self, morphology, reponame, sha1, filename): @@ -140,5 +138,4 @@ class MorphologyFactory(object): else: morphology.builds_artifacts = [morphology['name']] - morphology.needs_staging_area = True morphology.needs_artifact_metadata_cached = False diff --git a/morphlib/morphologyfactory_tests.py b/morphlib/morphologyfactory_tests.py index 6e17df48..30cfb8fb 100644 --- a/morphlib/morphologyfactory_tests.py +++ b/morphlib/morphologyfactory_tests.py @@ -223,18 +223,6 @@ class MorphologyFactoryTests(unittest.TestCase): morph = self.mf.get_morphology('reponame', 'sha1', 'system.morph') self.assertEqual(morph.builds_artifacts, ['system-rootfs']) - def test_sets_needs_staging_for_chunk(self): - morph = self.mf.get_morphology('reponame', 'sha1', 'chunk.morph') - self.assertEqual(morph.needs_staging_area, True) - - def test_does_not_set_needs_staging_for_stratum(self): - morph = self.mf.get_morphology('reponame', 'sha1', 'stratum.morph') - self.assertEqual(morph.needs_staging_area, False) - - def test_does_not_set_needs_staging_for_system(self): - morph = self.mf.get_morphology('reponame', 'sha1', 'system.morph') - self.assertEqual(morph.needs_staging_area, False) - def test_does_not_set_needs_artifact_metadata_cached_for_chunk(self): morph = self.mf.get_morphology('reponame', 'sha1', 'chunk.morph') self.assertEqual(morph.needs_artifact_metadata_cached, False) diff --git a/morphlib/plugins/trebuchet_plugin.py b/morphlib/plugins/trebuchet_plugin.py index 1ebffbf4..742d23c8 100644 --- a/morphlib/plugins/trebuchet_plugin.py +++ b/morphlib/plugins/trebuchet_plugin.py @@ -46,7 +46,8 @@ class TrebuchetPlugin(cliapp.Plugin): repo_name2, ref2, filename2 = args[4:7] app = self.app - build_env = morphlib.buildenvironment.BuildEnvironment(app.settings) + build_env = morphlib.buildenvironment.BuildEnvironment( + app.settings, morphlib.util.target(self.app.runcmd)) ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) lac, rac = morphlib.util.new_artifact_caches(app.settings) lrc, rrc = morphlib.util.new_repo_caches(app) diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index ee3e444f..24b72867 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -37,7 +37,8 @@ class StagingArea(object): _base_path = ['/sbin', '/usr/sbin', '/bin', '/usr/bin'] - def __init__(self, app, dirname, build_env, use_chroot=True, extra_env={}): + def __init__(self, app, dirname, build_env, use_chroot=True, extra_env={}, + extra_path=[]): self._app = app self.dirname = dirname self.builddirname = None @@ -52,7 +53,8 @@ class StagingArea(object): if use_chroot: path = build_env.extra_path + self._base_path else: - full_path = [self.relative(p) for p in build_env.extra_path] + rel_path = build_env.extra_path + full_path = [os.path.normpath(dirname + p) for p in rel_path] path = full_path + os.environ['PATH'].split(':') self.env['PATH'] = ':'.join(path) diff --git a/scripts/setup-3rd-party-strata b/scripts/setup-3rd-party-strata index a2a8b1e5..d1cc320d 100644 --- a/scripts/setup-3rd-party-strata +++ b/scripts/setup-3rd-party-strata @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -55,6 +55,7 @@ cat <<EOF > "$1/$2.morph" "name": "hello", "repo": "test:$2-hello", "ref": "master", + "build-mode": "test", "build-depends": [] } ] diff --git a/tests.as-root/build-with-external-strata.script b/tests.as-root/build-with-external-strata.script index 2d5d0fed..fd021399 100755 --- a/tests.as-root/build-with-external-strata.script +++ b/tests.as-root/build-with-external-strata.script @@ -38,6 +38,7 @@ cat <<EOF >> stratum2.morph "name": "linux", "repo": "test:kernel-repo", "ref": "master", + "build-mode": "test", "build-depends": [] } ] diff --git a/tests.as-root/setup b/tests.as-root/setup index a85507e0..b9d5d477 100755 --- a/tests.as-root/setup +++ b/tests.as-root/setup @@ -110,6 +110,7 @@ chunks: - name: hello repo: test:chunk-repo ref: farrokh + build-mode: test build-depends: [] EOF git add hello-stratum.morph @@ -125,6 +126,7 @@ chunks: - name: tools repo: test:tools-repo ref: master + build-mode: test build-depends: [] EOF git add tools-stratum.morph @@ -153,6 +155,7 @@ chunks: - name: linux repo: test:kernel-repo ref: master + build-mode: test build-depends: [] EOF git add linux-stratum.morph diff --git a/tests.as-root/system-overlap.script b/tests.as-root/system-overlap.script index cc308536..b8888491 100755 --- a/tests.as-root/system-overlap.script +++ b/tests.as-root/system-overlap.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011, 2012 Codethink Limited +# Copyright (C) 2011-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -57,12 +57,14 @@ cat <<EOF >foo-baz-stratum.morph "name": "overlap-foo-baz", "repo": "test:chunk-repo", "ref": "overlap", + "build-mode": "test", "build-depends": [] }, { "name": "linux", "repo": "test:kernel-repo", "ref": "master", + "build-mode": "test", "build-depends": ["overlap-foo-baz"] } ] @@ -77,12 +79,14 @@ cat <<EOF >foo-barqux-stratum.morph "name": "overlap-foobar", "repo": "test:chunk-repo", "ref": "overlap", + "build-mode": "test", "build-depends": [] }, { "name": "overlap-fooqux", "repo": "test:chunk-repo", "ref": "overlap", + "build-mode": "test", "build-depends": ["overlap-foobar"] } ] diff --git a/tests.as-root/tarball-image-is-sensible.setup b/tests.as-root/tarball-image-is-sensible.setup index e159070c..fa904c2c 100755 --- a/tests.as-root/tarball-image-is-sensible.setup +++ b/tests.as-root/tarball-image-is-sensible.setup @@ -84,6 +84,7 @@ cat <<EOF > link-stratum.morph "name": "links", "repo": "test:chunk-repo", "ref": "tarball-links", + "build-mode": "test", "build-depends": [] } ] diff --git a/tests.branching/workflow-petrify.stdout b/tests.branching/workflow-petrify.stdout index b572ed67..c86afa2e 100644 --- a/tests.branching/workflow-petrify.stdout +++ b/tests.branching/workflow-petrify.stdout @@ -31,6 +31,7 @@ test/petrify after petrifying: "name": "hello", "repo": "test:stratum2-hello", "ref": "f4730636e429149bb923fa16be3aa9802d484b23", + "build-mode": "test", "build-depends": [], "unpetrify-ref": "master" } @@ -44,6 +45,7 @@ test/petrify after petrifying: "name": "hello", "repo": "test:stratum3-hello", "ref": "f4730636e429149bb923fa16be3aa9802d484b23", + "build-mode": "test", "build-depends": [], "unpetrify-ref": "master" } @@ -83,6 +85,7 @@ test/petrify after editing a chunk: "name": "hello", "repo": "test:stratum2-hello", "ref": "test/petrify", + "build-mode": "test", "build-depends": [] } ] @@ -95,6 +98,7 @@ test/petrify after editing a chunk: "name": "hello", "repo": "test:stratum3-hello", "ref": "f4730636e429149bb923fa16be3aa9802d484b23", + "build-mode": "test", "build-depends": [], "unpetrify-ref": "master" } @@ -134,6 +138,7 @@ test/unpetrify after unpetrifying: "name": "hello", "repo": "test:stratum2-hello", "ref": "test/petrify", + "build-mode": "test", "build-depends": [] } ] @@ -146,6 +151,7 @@ test/unpetrify after unpetrifying: "name": "hello", "repo": "test:stratum3-hello", "ref": "master", + "build-mode": "test", "build-depends": [] } ] diff --git a/tests.build/build-stratum-with-submodules.script b/tests.build/build-stratum-with-submodules.script index f64ba9f6..c3c00578 100755 --- a/tests.build/build-stratum-with-submodules.script +++ b/tests.build/build-stratum-with-submodules.script @@ -55,7 +55,8 @@ cat <<EOF > "$morphs/hello-stratum.morph" "name": "parent", "repo": "test:parent-repo", "ref": "master", - "build-depends": [] + "build-depends": [], + "build-mode": "test" } ] } diff --git a/tests.build/setup b/tests.build/setup index 935e388b..499dbb21 100755 --- a/tests.build/setup +++ b/tests.build/setup @@ -95,6 +95,7 @@ cat <<EOF > hello-stratum.morph "name": "hello", "repo": "test:chunk-repo", "ref": "farrokh", + "build-mode": "test", "build-depends": [] } ] diff --git a/tests.build/stratum-overlap-warns.setup b/tests.build/stratum-overlap-warns.setup index 520a37a1..626f2094 100755 --- a/tests.build/stratum-overlap-warns.setup +++ b/tests.build/stratum-overlap-warns.setup @@ -34,25 +34,29 @@ cat <<EOF >hello-stratum.morph "name": "dirs", "repo": "test:chunk-repo", "ref": "overlap", - "build-depends": [] + "build-depends": [], + "build-mode": "test" }, { "name": "overlap-foobar", "repo": "test:chunk-repo", "ref": "overlap", - "build-depends": ["dirs"] + "build-depends": ["dirs"], + "build-mode": "test" }, { "name": "overlap-fooqux", "repo": "test:chunk-repo", "ref": "overlap", - "build-depends": ["overlap-foobar"] + "build-depends": ["overlap-foobar"], + "build-mode": "test" }, { "name": "overlap-foo-baz", "repo": "test:chunk-repo", "ref": "overlap", - "build-depends": ["overlap-fooqux"] + "build-depends": ["overlap-fooqux"], + "build-mode": "test" } ] } diff --git a/tests.deploy/setup b/tests.deploy/setup index b0054fa5..584ce039 100755 --- a/tests.deploy/setup +++ b/tests.deploy/setup @@ -100,7 +100,8 @@ cat <<EOF > hello-stratum.morph "name": "hello", "repo": "test:chunk-repo", "ref": "farrokh", - "build-depends": [] + "build-depends": [], + "build-mode": "test" } ] } @@ -140,7 +141,8 @@ cat <<EOF > linux-stratum.morph "name": "linux", "repo": "test:kernel-repo", "ref": "master", - "build-depends": [] + "build-depends": [], + "build-mode": "test" } ] } |