From 84807d4d7c23f45d4f0a0f87e6c7ba7ba7470936 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Wed, 6 Feb 2013 10:23:57 +0000 Subject: Add 'build-mode' field for chunks in a stratum Allowed values: staging: build with a staging chroot (default) test: build with the host's tools bootstrap: build with the host's tools, and do not include this chunk in the final stratum artifact In the past, 'normal mode' has been used to describe building a chunk with the host's tools. We don't want that mode to ever be used, because it is a huge hole in reproducability, but we need to keep it around to avoid making Morph's cmdtest suite depend on Baserock. Hopefully naming it 'test' should discourage potential abusers. It is unfortunate that the build tests now take a separate code path compared to real-world usage of Morph. However, this is necessary to avoid a circular dependency between Morph's test suite and the build-essential stratum in Baserock. We do whole-build testing of Baserock, too, so the 'staging' code path is still tested outside of Morph. However, testing a staging area requires populating it with at minimum a working shell, and this is a bit too complex to go in Morph's test suite. --- morphlib/artifactresolver.py | 5 ++++- morphlib/buildcommand.py | 38 ++++++++++++++++++++++++------------ morphlib/buildenvironment.py | 12 +++++++++++- morphlib/cachekeycomputer.py | 1 + morphlib/morph2.py | 4 +++- morphlib/morphologyfactory.py | 3 --- morphlib/morphologyfactory_tests.py | 12 ------------ morphlib/plugins/trebuchet_plugin.py | 3 ++- morphlib/stagingarea.py | 6 ++++-- 9 files changed, 51 insertions(+), 33 deletions(-) (limited to 'morphlib') diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index 4b7956e0..76178b35 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -220,6 +220,9 @@ class ArtifactResolver(object): for other_stratum in strata: chunk_artifact.add_dependency(other_stratum) + # Resolve now to avoid a search for the parent morphology later + chunk_source.build_mode = info['build-mode'] + build_depends = info.get('build-depends', None) if build_depends is None: diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index ca097145..2de71d8f 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -226,13 +226,24 @@ class BuildCommand(object): self.get_sources(artifact) deps = self.get_recursive_deps(artifact) self.cache_artifacts_locally(deps) - staging_area = self.create_staging_area() - if artifact.source.morphology.needs_staging_area: + + setup_mounts = False + if artifact.source.morphology['kind'] == 'chunk': + build_mode = artifact.source.build_mode + + if build_mode not in ['bootstrap', 'staging', 'test']: + raise morphlib.Error( + 'Unknown build mode for chunk %s: %s' % + (artifact.name, build_mode)) + + use_chroot = build_mode=='staging' + staging_area = self.create_staging_area(use_chroot) self.install_fillers(staging_area) - self.install_chunk_artifacts(staging_area, deps, artifact) - morphlib.builder2.ldconfig(self.app.runcmd, - staging_area.dirname) - self.build_and_cache(staging_area, artifact) + self.install_dependencies(staging_area, deps, artifact) + else: + staging_area = self.create_staging_area() + + self.build_and_cache(staging_area, artifact, setup_mounts) self.remove_staging_area(staging_area) def get_recursive_deps(self, artifact): @@ -301,13 +312,13 @@ class BuildCommand(object): copy(self.rac.get_artifact_metadata(artifact, 'meta'), self.lac.put_artifact_metadata(artifact, 'meta')) - def create_staging_area(self): + def create_staging_area(self, use_chroot=True): '''Create the staging area for building a single artifact.''' self.app.status(msg='Creating staging area') staging_dir = tempfile.mkdtemp(dir=self.app.settings['tempdir']) staging_area = morphlib.stagingarea.StagingArea( - self.app, staging_dir, self.build_env, False, {}) + self.app, staging_dir, self.build_env, use_chroot, {}) return staging_area def remove_staging_area(self, staging_area): @@ -328,7 +339,7 @@ class BuildCommand(object): filename=filename) staging_area.install_artifact(f) - def install_chunk_artifacts(self, staging_area, artifacts, parent_art): + def install_dependencies(self, staging_area, artifacts, target_artifact): '''Install chunk artifacts into staging area. We only ever care about chunk artifacts as build dependencies, @@ -343,12 +354,15 @@ class BuildCommand(object): if artifact.source.morphology['kind'] != 'chunk': continue self.app.status(msg='[%(name)s] Installing chunk %(chunk_name)s', - name=parent_art.name, + name=target_artifact.name, chunk_name=artifact.name) handle = self.lac.get(artifact) staging_area.install_artifact(handle) - def build_and_cache(self, staging_area, artifact): + if target_artifact.source.build_mode == 'staging': + morphlib.builder2.ldconfig(self.app.runcmd, staging_area.dirname) + + def build_and_cache(self, staging_area, artifact, setup_mounts): '''Build an artifact and put it into the local artifact cache.''' self.app.status(msg='Starting actual build: %(name)s', @@ -356,5 +370,5 @@ class BuildCommand(object): setup_mounts = self.app.settings['staging-chroot'] builder = morphlib.builder2.Builder( self.app, staging_area, self.lac, self.rac, self.lrc, - self.app.settings['max-jobs'], True) + self.app.settings['max-jobs'], setup_mounts) return builder.build_and_cache(artifact) diff --git a/morphlib/buildenvironment.py b/morphlib/buildenvironment.py index 29561220..6ba950ff 100644 --- a/morphlib/buildenvironment.py +++ b/morphlib/buildenvironment.py @@ -13,6 +13,7 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import copy import cliapp import os @@ -21,6 +22,16 @@ import morphlib class BuildEnvironment(): + '''Represents the build environment for an artifact + + This should be as consistent as possible across builds, but some + artifacts will require tweaks. The intention of this object is + to create one once and call populate() to create an initial state + and when changes are required, call clone() to get another instance + which can be modified. + + ''' + def __init__(self, settings, target, arch=None): '''Create a new BuildEnvironment object''' @@ -88,7 +99,6 @@ class BuildEnvironment(): if not settings['no-ccache']: self.extra_path.append(self._ccache_path) - # FIXME: we should set CCACHE_BASEDIR so any objects that refer to their # current directory get corrected. This improve the cache hit rate # env['CCACHE_BASEDIR'] = self.tempdir.dirname diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index d9ad5762..4573ad0d 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -87,6 +87,7 @@ class CacheKeyComputer(object): kind = artifact.source.morphology['kind'] if kind == 'chunk': + keys['build-mode'] = artifact.source.build_mode keys['tree'] = artifact.source.tree elif kind in ('system', 'stratum'): morphology = artifact.source.morphology diff --git a/morphlib/morph2.py b/morphlib/morph2.py index 3cdf49a9..728fa533 100644 --- a/morphlib/morph2.py +++ b/morphlib/morph2.py @@ -52,7 +52,7 @@ class Morphology(object): 'stratum': [ ('chunks', []), ('description', ''), - ('build-depends', None) + ('build-depends', None), ], 'system': [ ('strata', []), @@ -157,6 +157,8 @@ class Morphology(object): self._set_default_value(source, 'morph', source['name']) if 'build-depends' not in source: self._set_default_value(source, 'build-depends', None) + if 'build-mode' not in source: + self._set_default_value(source, 'build-mode', 'staging') def _parse_size(self, size): if isinstance(size, basestring): diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py index 76905eb9..7ae68697 100644 --- a/morphlib/morphologyfactory.py +++ b/morphlib/morphologyfactory.py @@ -113,7 +113,6 @@ class MorphologyFactory(object): name = morphology['name'] morphology.builds_artifacts = [name + '-rootfs'] - morphology.needs_staging_area = False morphology.needs_artifact_metadata_cached = False def _check_and_tweak_stratum(self, morphology, reponame, sha1, filename): @@ -129,7 +128,6 @@ class MorphologyFactory(object): (filename, name)) morphology.builds_artifacts = [morphology['name']] - morphology.needs_staging_area = False morphology.needs_artifact_metadata_cached = True def _check_and_tweak_chunk(self, morphology, reponame, sha1, filename): @@ -140,5 +138,4 @@ class MorphologyFactory(object): else: morphology.builds_artifacts = [morphology['name']] - morphology.needs_staging_area = True morphology.needs_artifact_metadata_cached = False diff --git a/morphlib/morphologyfactory_tests.py b/morphlib/morphologyfactory_tests.py index 6e17df48..30cfb8fb 100644 --- a/morphlib/morphologyfactory_tests.py +++ b/morphlib/morphologyfactory_tests.py @@ -223,18 +223,6 @@ class MorphologyFactoryTests(unittest.TestCase): morph = self.mf.get_morphology('reponame', 'sha1', 'system.morph') self.assertEqual(morph.builds_artifacts, ['system-rootfs']) - def test_sets_needs_staging_for_chunk(self): - morph = self.mf.get_morphology('reponame', 'sha1', 'chunk.morph') - self.assertEqual(morph.needs_staging_area, True) - - def test_does_not_set_needs_staging_for_stratum(self): - morph = self.mf.get_morphology('reponame', 'sha1', 'stratum.morph') - self.assertEqual(morph.needs_staging_area, False) - - def test_does_not_set_needs_staging_for_system(self): - morph = self.mf.get_morphology('reponame', 'sha1', 'system.morph') - self.assertEqual(morph.needs_staging_area, False) - def test_does_not_set_needs_artifact_metadata_cached_for_chunk(self): morph = self.mf.get_morphology('reponame', 'sha1', 'chunk.morph') self.assertEqual(morph.needs_artifact_metadata_cached, False) diff --git a/morphlib/plugins/trebuchet_plugin.py b/morphlib/plugins/trebuchet_plugin.py index 1ebffbf4..742d23c8 100644 --- a/morphlib/plugins/trebuchet_plugin.py +++ b/morphlib/plugins/trebuchet_plugin.py @@ -46,7 +46,8 @@ class TrebuchetPlugin(cliapp.Plugin): repo_name2, ref2, filename2 = args[4:7] app = self.app - build_env = morphlib.buildenvironment.BuildEnvironment(app.settings) + build_env = morphlib.buildenvironment.BuildEnvironment( + app.settings, morphlib.util.target(self.app.runcmd)) ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) lac, rac = morphlib.util.new_artifact_caches(app.settings) lrc, rrc = morphlib.util.new_repo_caches(app) diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index ee3e444f..24b72867 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -37,7 +37,8 @@ class StagingArea(object): _base_path = ['/sbin', '/usr/sbin', '/bin', '/usr/bin'] - def __init__(self, app, dirname, build_env, use_chroot=True, extra_env={}): + def __init__(self, app, dirname, build_env, use_chroot=True, extra_env={}, + extra_path=[]): self._app = app self.dirname = dirname self.builddirname = None @@ -52,7 +53,8 @@ class StagingArea(object): if use_chroot: path = build_env.extra_path + self._base_path else: - full_path = [self.relative(p) for p in build_env.extra_path] + rel_path = build_env.extra_path + full_path = [os.path.normpath(dirname + p) for p in rel_path] path = full_path + os.environ['PATH'].split(':') self.env['PATH'] = ':'.join(path) -- cgit v1.2.1