diff options
Diffstat (limited to 'morphlib')
26 files changed, 476 insertions, 376 deletions
diff --git a/morphlib/app.py b/morphlib/app.py index 87710ee5..ac36353e 100755 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -41,8 +41,6 @@ defaults = { ], 'cachedir': os.path.expanduser('~/.cache/morph'), 'max-jobs': morphlib.util.make_concurrency(), - 'prefix': '/usr', - 'toolchain-target': '%s-baserock-linux-gnu' % os.uname()[4], 'build-ref-prefix': 'baserock/builds' } @@ -105,16 +103,7 @@ class Morph(cliapp.Application): default=None, group=group_advanced) - # Build Options group_build = 'Build Options' - self.settings.boolean(['bootstrap'], - 'build stuff in bootstrap mode; this is ' - 'DANGEROUS and will install stuff on your ' - 'system', - group=group_build) - self.settings.boolean(['keep-path'], - 'do not touch the PATH environment variable', - group=group_build) self.settings.integer(['max-jobs'], 'run at most N parallel jobs with make (default ' 'is to a value based on the number of CPUs ' @@ -127,28 +116,10 @@ class Morph(cliapp.Application): self.settings.boolean(['no-distcc'], 'do not use distcc (default: true)', group=group_build, default=True) - self.settings.string(['prefix'], - 'build chunks with prefix PREFIX', - metavar='PREFIX', default=defaults['prefix'], - group=group_build) self.settings.boolean(['push-build-branches'], 'always push temporary build branches to the ' 'remote repository', group=group_build) - self.settings.boolean(['staging-chroot'], - 'build things in an isolated chroot ' - '(default: true)', - group=group_build) - self.settings.string_list(['staging-filler'], - 'use FILE as contents of build chroot', - metavar='FILE', - group=group_build) - self.settings.string(['target-cflags'], - 'inject additional CFLAGS into the environment ' - 'that is used to build chunks', - metavar='CFLAGS', - default='', - group=group_build) self.settings.string(['tempdir'], 'temporary directory to use for builds ' '(this is separate from just setting $TMPDIR ' @@ -159,13 +130,19 @@ class Morph(cliapp.Application): metavar='DIR', default=os.environ.get('TMPDIR'), group=group_build) - self.settings.string(['toolchain-target'], - 'set the TOOLCHAIN_TARGET variable which is used ' - 'in some chunks to determine which architecture ' - 'to build tools for', - metavar='TOOLCHAIN_TARGET', - default=defaults['toolchain-target'], - group=group_build) + + # These cannot be removed just yet because existing morph.conf files + # would fail to parse. + group_obsolete = 'Obsolete Options' + self.settings.boolean(['staging-chroot'], + 'build things in an isolated chroot ' + '(default: true)', + default=True, + group=group_obsolete) + self.settings.string_list(['staging-filler'], + 'use FILE as contents of build chroot', + metavar='FILE', + group=group_obsolete) def check_time(self): # Check that the current time is not far in the past. @@ -179,6 +156,18 @@ class Morph(cliapp.Application): def process_args(self, args): self.check_time() + # Handle obsolete options + if self.settings['staging-chroot'] is not True: + raise cliapp.AppException( + 'The "staging-chroot" option has been set to False. This ' + 'option is obsolete and should be left as the default (True).') + if self.settings['staging-filler'] is not None: + self.status(msg='WARNING! A staging filler was specified. Staging ' + 'fillers are deprecated and may break new builds. You ' + 'should only specify this option if you are building ' + 'a system based on a version of Baserock older than ' + 'Baserock 6.') + # Combine the aliases into repo-alias before passing on to normal # command processing. This means everything from here on down can # treat settings['repo-alias'] as the sole source of prefixes for git diff --git a/morphlib/artifact.py b/morphlib/artifact.py index aef48d76..82680709 100644 --- a/morphlib/artifact.py +++ b/morphlib/artifact.py @@ -61,6 +61,20 @@ class Artifact(object): self.name, metadata_name) + def get_dependency_prefix_set(self): + '''Collects all install prefixes of this artifact's build dependencies + + If any of the build dependencies of a chunk artifact are installed + to non-standard prefixes, we need to add those prefixes to the + PATH of the current artifact. + + ''' + result = set() + for d in self.dependencies: + if d.source.morphology['kind'] == 'chunk': + result.add(d.source.prefix) + return result + def __str__(self): # pragma: no cover return '%s|%s' % (self.source, self.name) @@ -83,4 +97,3 @@ class Artifact(object): yield a return list(depth_first(self)) - diff --git a/morphlib/artifact_tests.py b/morphlib/artifact_tests.py index 1d9e6cca..8edbbde2 100644 --- a/morphlib/artifact_tests.py +++ b/morphlib/artifact_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,6 +14,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import copy import unittest import morphlib @@ -78,3 +79,12 @@ class ArtifactTests(unittest.TestCase): self.assertEqual(self.artifact.dependencies, [self.other]) self.assertEqual(self.other.dependents, [self.artifact]) self.assertTrue(self.artifact.depends_on(self.other)) + + def test_get_dependency_prefix(self): + self.artifact.add_dependency(self.other) + self.artifact.source.prefix = '/bar' + self.other.source = copy.copy(self.artifact.source) + self.other.source.prefix = '/foo' + + prefix_set = self.artifact.get_dependency_prefix_set() + self.assertEqual(prefix_set, set(['/foo'])) diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index 4b7956e0..186d5357 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -220,6 +220,10 @@ class ArtifactResolver(object): for other_stratum in strata: chunk_artifact.add_dependency(other_stratum) + # Resolve now to avoid a search for the parent morphology later + chunk_source.build_mode = info['build-mode'] + chunk_source.prefix = info['prefix'] + build_depends = info.get('build-depends', None) if build_depends is None: diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index dfacd760..d49f9e72 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -33,12 +33,11 @@ class BuildCommand(object): ''' def __init__(self, app): + self.supports_local_build = True + self.app = app - self.build_env = self.new_build_env() - self.ckc = self.new_cache_key_computer(self.build_env) self.lac, self.rac = self.new_artifact_caches() self.lrc, self.rrc = self.new_repo_caches() - self.supports_local_build = True def build(self, args): '''Build triplets specified on command line.''' @@ -53,14 +52,6 @@ class BuildCommand(object): self.app.status(msg='Build ends successfully') - def new_build_env(self): - '''Create a new BuildEnvironment instance.''' - return morphlib.buildenvironment.BuildEnvironment(self.app.settings) - - def new_cache_key_computer(self, build_env): - '''Create a new cache key computer.''' - return morphlib.cachekeycomputer.CacheKeyComputer(build_env) - def new_artifact_caches(self): '''Create interfaces for the build artifact caches. @@ -72,6 +63,11 @@ class BuildCommand(object): def new_repo_caches(self): return morphlib.util.new_repo_caches(self.app) + def new_build_env(self, arch): + '''Create a new BuildEnvironment instance.''' + return morphlib.buildenvironment.BuildEnvironment(self.app.settings, + arch) + def get_artifact_object(self, repo_name, ref, filename): '''Create an Artifact object representing the given triplet.''' @@ -81,11 +77,6 @@ class BuildCommand(object): srcpool = self.app.create_source_pool( self.lrc, self.rrc, (repo_name, ref, filename)) - root_kind = srcpool.lookup(repo_name, ref, filename).morphology['kind'] - if root_kind not in ['system', 'stratum']: - raise morphlib.Error( - 'Building a %s directly is not supported' % root_kind) - self.app.status( msg='Validating cross-morphology references', chatty=True) self._validate_cross_morphology_references(srcpool) @@ -96,14 +87,26 @@ class BuildCommand(object): self.app.status(msg='Resolving artifacts', chatty=True) artifacts = ar.resolve_artifacts(srcpool) - self.app.status(msg='Computing cache keys', chatty=True) - for artifact in artifacts: - artifact.cache_key = self.ckc.compute_key(artifact) - artifact.cache_id = self.ckc.get_cache_id(artifact) - self.app.status(msg='Computing build order', chatty=True) root_artifact = self._find_root_artifact(artifacts) + root_kind = root_artifact.source.morphology['kind'] + if root_kind != 'system': + raise morphlib.Error( + 'Building a %s directly is not supported' % root_kind) + arch = root_artifact.source.morphology['arch'] + + self.app.status(msg='Creating build environment for %(arch)s', + arch=arch, chatty=True) + build_env = self.new_build_env(arch) + + self.app.status(msg='Computing cache keys', chatty=True) + ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) + for artifact in artifacts: + artifact.cache_key = ckc.compute_key(artifact) + artifact.cache_id = ckc.get_cache_id(artifact) + + root_artifact.build_env = build_env return root_artifact def _validate_cross_morphology_references(self, srcpool): @@ -181,77 +184,81 @@ class BuildCommand(object): assert len(maybe) == 1 return maybe.pop() - def build_in_order(self, artifact): + def build_in_order(self, root_artifact): '''Build everything specified in a build order.''' - self.app.status(msg='Building according to build ordering', - chatty=True) - artifacts = artifact.walk() + self.app.status(msg='Building a set of artifacts', chatty=True) + build_env = root_artifact.build_env + artifacts = root_artifact.walk() old_prefix = self.app.status_prefix for i, a in enumerate(artifacts): self.app.status_prefix = ( old_prefix + '[Build %d/%d] ' % ((i+1), len(artifacts))) - self.build_artifact(a) + + self.app.status(msg='[%(name)s] Checking if %(kind)s needs ' + 'building %(sha1)s', + name=a.name, kind=a.source.morphology['kind'], + sha1=a.source.sha1[:7]) + + if self.is_built(a): + self.app.status(msg='[%(name)s] The %(kind)s is already built', + name=a.name, kind=a.source.morphology['kind']) + self.cache_artifacts_locally([a]) + else: + self.app.status(msg='[%(name)s] Building %(kind)s %(name)s', + name=a.name, kind=a.source.morphology['kind']) + self.build_artifact(a, build_env) + + self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s', + kind=a.source.morphology['kind'], name=a.name, + cachepath=self.lac.artifact_filename(a), + chatty=(a.source.morphology['kind'] != "system")) self.app.status_prefix = old_prefix - def build_artifact(self, artifact): + def is_built(self, artifact): + '''Does either cache already have the artifact?''' + return self.lac.has(artifact) or (self.rac and self.rac.has(artifact)) + + def build_artifact(self, artifact, build_env): '''Build one artifact. All the dependencies are assumed to be built and available in either the local or remote cache already. ''' - - self.app.status(msg='Checking if %(kind)s %(name)s needs building', - kind=artifact.source.morphology['kind'], - name=artifact.name) - - if self.is_built(artifact): - self.app.status(msg='The %(kind)s %(name)s is already built', - kind=artifact.source.morphology['kind'], - name=artifact.name) - self.cache_artifacts_locally([artifact]) + self.get_sources(artifact) + deps = self.get_recursive_deps(artifact) + self.cache_artifacts_locally(deps) + + setup_mounts = False + if artifact.source.morphology['kind'] == 'chunk': + build_mode = artifact.source.build_mode + extra_env = {'PREFIX': artifact.source.prefix} + + dep_prefix_set = artifact.get_dependency_prefix_set() + extra_path = [os.path.join(d, 'bin') for d in dep_prefix_set] + + if build_mode not in ['bootstrap', 'staging', 'test']: + logging.warning('Unknown build mode %s for chunk %s. ' + 'Defaulting to staging mode.' % + (build_mode, artifact.name)) + build_mode = 'staging' + + use_chroot = build_mode=='staging' + staging_area = self.create_staging_area(build_env, + use_chroot, + extra_env=extra_env, + extra_path=extra_path) + self.install_fillers(staging_area) + self.install_dependencies(staging_area, deps, artifact) else: - self.app.status(msg='Building %(kind)s %(name)s', - kind=artifact.source.morphology['kind'], - name=artifact.name) - self.get_sources(artifact) - deps = self.get_recursive_deps(artifact) - self.cache_artifacts_locally(deps) - staging_area = self.create_staging_area(artifact) - if self.app.settings['staging-chroot']: - if artifact.source.morphology.needs_staging_area: - self.install_fillers(staging_area) - self.install_chunk_artifacts(staging_area, - deps, artifact) - morphlib.builder2.ldconfig(self.app.runcmd, - staging_area.tempdir) - - self.build_and_cache(staging_area, artifact) - if self.app.settings['bootstrap']: - self.install_chunk_artifacts(staging_area, - (artifact,)) - self.remove_staging_area(staging_area) - self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s', - kind=artifact.source.morphology['kind'], - name=artifact.name, - cachepath=self.lac.artifact_filename(artifact), - chatty=(artifact.source.morphology['kind'] != - "system")) + staging_area = self.create_staging_area(build_env, False) - def is_built(self, artifact): - '''Does either cache already have the artifact?''' - return self.lac.has(artifact) or (self.rac and self.rac.has(artifact)) + self.build_and_cache(staging_area, artifact, setup_mounts) + self.remove_staging_area(staging_area) def get_recursive_deps(self, artifact): - done = set() - todo = set((artifact,)) - while todo: - for a in todo.pop().dependencies: - if a not in done: - done.add(a) - todo.add(a) - return done + return artifact.walk()[:-1] def get_sources(self, artifact): '''Update the local git repository cache with the sources.''' @@ -316,39 +323,25 @@ class BuildCommand(object): copy(self.rac.get_artifact_metadata(artifact, 'meta'), self.lac.put_artifact_metadata(artifact, 'meta')) - def create_staging_area(self, artifact): + def create_staging_area(self, build_env, use_chroot=True, extra_env={}, + extra_path=[]): '''Create the staging area for building a single artifact.''' - if self.app.settings['staging-chroot']: - staging_root = tempfile.mkdtemp(dir=self.app.settings['tempdir']) - staging_temp = staging_root - else: - staging_root = '/' - staging_temp = tempfile.mkdtemp(dir=self.app.settings['tempdir']) - self.app.status(msg='Creating staging area') - staging_area = morphlib.stagingarea.StagingArea(self.app, - staging_root, - staging_temp) + staging_dir = tempfile.mkdtemp(dir=self.app.settings['tempdir']) + staging_area = morphlib.stagingarea.StagingArea( + self.app, staging_dir, build_env, use_chroot, extra_env, + extra_path) return staging_area def remove_staging_area(self, staging_area): '''Remove the staging area.''' - if staging_area.dirname != '/': - self.app.status(msg='Removing staging area') - staging_area.remove() - temp_path = staging_area.tempdir - if temp_path != '/' and os.path.exists(temp_path): - self.app.status(msg='Removing temporary staging directory') - shutil.rmtree(temp_path) + self.app.status(msg='Removing staging area') + staging_area.remove() def install_fillers(self, staging_area): - '''Install staging fillers into the staging area. - - This must not be called in bootstrap mode. - - ''' + '''Install staging fillers into the staging area.''' logging.debug('Pre-populating staging area %s' % staging_area.dirname) logging.debug('Fillers: %s' % @@ -359,7 +352,16 @@ class BuildCommand(object): filename=filename) staging_area.install_artifact(f) - def install_chunk_artifacts(self, staging_area, artifacts, parent_art): + # Nasty hack to avoid installing chunks built in 'bootstrap' mode in a + # different stratum when constructing staging areas. + def is_stratum(self, a): + return a.source.morphology['kind'] == 'stratum' + + def in_same_stratum(self, a, b): + return len(filter(self.is_stratum, a.dependencies)) == \ + len(filter(self.is_stratum, b.dependencies)) + + def install_dependencies(self, staging_area, artifacts, target_artifact): '''Install chunk artifacts into staging area. We only ever care about chunk artifacts as build dependencies, @@ -373,19 +375,26 @@ class BuildCommand(object): for artifact in artifacts: if artifact.source.morphology['kind'] != 'chunk': continue + if artifact.source.build_mode == 'bootstrap': + if not self.in_same_stratum(artifact, target_artifact): + continue self.app.status(msg='[%(name)s] Installing chunk %(chunk_name)s', - name=parent_art.name, + name=target_artifact.name, chunk_name=artifact.name) handle = self.lac.get(artifact) staging_area.install_artifact(handle) - def build_and_cache(self, staging_area, artifact): + if target_artifact.source.build_mode == 'staging': + morphlib.builder2.ldconfig(self.app.runcmd, staging_area.dirname) + + def build_and_cache(self, staging_area, artifact, setup_mounts): '''Build an artifact and put it into the local artifact cache.''' - self.app.status(msg='Starting actual build: %(name)s', - name=artifact.name) + self.app.status(msg='[%(name)s] Starting actual build: %(name)s ' + '%(sha1)s', + name=artifact.name, sha1=artifact.source.sha1[:7]) setup_mounts = self.app.settings['staging-chroot'] builder = morphlib.builder2.Builder( self.app, staging_area, self.lac, self.rac, self.lrc, - self.build_env, self.app.settings['max-jobs'], setup_mounts) + self.app.settings['max-jobs'], setup_mounts) return builder.build_and_cache(artifact) diff --git a/morphlib/buildenvironment.py b/morphlib/buildenvironment.py index d9e3210f..68e7e756 100644 --- a/morphlib/buildenvironment.py +++ b/morphlib/buildenvironment.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -13,6 +13,8 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import copy +import cliapp import os import morphlib @@ -20,18 +22,32 @@ import morphlib class BuildEnvironment(): - def __init__(self, settings, arch=None): - self.arch = morphlib.util.arch() if arch is None else arch + '''Represents the build environment for an artifact + + This should be as consistent as possible across builds, but some + artifacts will require tweaks. The intention of this object is + to create one once and call populate() to create an initial state + and when changes are required, call clone() to get another instance + which can be modified. + + ''' + + def __init__(self, settings, arch): + '''Create a new BuildEnvironment object''' + + self.extra_path = [] + self.env = self._clean_env(settings) + self.env.update(self._env_for_arch(arch)) + _osenv = os.environ - _default_path = '/sbin:/usr/sbin:/bin:/usr/bin' - _override_term = 'dumb' + _ccache_path = '/usr/lib/ccache' + _override_home = '/tmp' + _override_locale = 'C' _override_shell = '/bin/sh' + _override_term = 'dumb' _override_username = 'tomjon' - _override_locale = 'C' - _override_home = '/tmp' - _ccache_path = '/usr/lib/ccache' def _clean_env(self, settings): '''Create a fresh set of environment variables for a clean build. @@ -40,11 +56,8 @@ class BuildEnvironment(): ''' - path = self._osenv['PATH'] - # copy a set of white-listed variables from the original env copied_vars = dict.fromkeys([ - 'BOOTSTRAP_TOOLS', 'DISTCC_HOSTS', 'LD_PRELOAD', 'LD_LIBRARY_PATH', @@ -62,10 +75,6 @@ class BuildEnvironment(): if copied_vars[name] is not None: env[name] = copied_vars[name] - if settings['bootstrap'] or not settings['staging-chroot']: - if 'TMPDIR' in self._osenv: - env['TMPDIR'] = self._osenv['TMPDIR'] - env['TERM'] = self._override_term env['SHELL'] = self._override_shell env['USER'] = \ @@ -74,17 +83,8 @@ class BuildEnvironment(): env['LC_ALL'] = self._override_locale env['HOME'] = self._override_home - if settings['keep-path'] or settings['bootstrap']: - env['PATH'] = path - else: - env['PATH'] = self._default_path - - env['TOOLCHAIN_TARGET'] = settings['toolchain-target'] - env['CFLAGS'] = settings['target-cflags'] - env['PREFIX'] = settings['prefix'] - env['BOOTSTRAP'] = 'true' if settings['bootstrap'] else 'false' if not settings['no-ccache']: - env['PATH'] = ('%s:%s' % (self._ccache_path, env['PATH'])) + self.extra_path.append(self._ccache_path) # FIXME: we should set CCACHE_BASEDIR so any objects that refer to their # current directory get corrected. This improve the cache hit rate # env['CCACHE_BASEDIR'] = self.tempdir.dirname @@ -98,3 +98,32 @@ class BuildEnvironment(): env['CCACHE_PREFIX'] = 'distcc' return env + + def _env_for_arch(self, arch): + '''Set build environment variables specific to the target machine + + These are entirely controlled by the 'arch' field in the system + morphology, which is passed to the morphologies as MORPH_ARCH to + do what they like with. + + ''' + + env = {} + env['MORPH_ARCH'] = arch + + # GNU triplets are widely used, so we handle these in Morph rather + # than leaving it up to individual morphologies. + if arch == 'x86_32': + cpu = 'i686' + else: + cpu = arch + + if arch.startswith('arm'): + abi = 'eabi' + else: + abi = '' + + env['TARGET'] = cpu + '-baserock-linux-gnu' + abi + env['TARGET_STAGE1'] = cpu + '-bootstrap-linux-gnu' + abi + + return env diff --git a/morphlib/buildenvironment_tests.py b/morphlib/buildenvironment_tests.py index 61844c19..7ae7c2d5 100644 --- a/morphlib/buildenvironment_tests.py +++ b/morphlib/buildenvironment_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,6 +14,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import copy import unittest import morphlib @@ -24,63 +25,27 @@ class BuildEnvironmentTests(unittest.TestCase): def setUp(self): self.settings = { - 'keep-path': False, - 'bootstrap': False, - 'toolchain-target': '%s-baserock-linux-gnu' % morphlib.util.arch(), - 'target-cflags': '', 'prefix': '/usr', 'no-ccache': True, - 'no-distcc': True, - 'staging-chroot': False, + 'no-distcc': True } self.fake_env = { 'PATH': '/fake_bin', } - self.default_path = 'no:such:path' - def test_arch_defaults_to_host(self): - buildenv = buildenvironment.BuildEnvironment(self.settings) - self.assertEqual(buildenv.arch, morphlib.util.arch()) + def new_build_env(self, settings=None, target=None, **kws): + settings = settings or self.settings + target = target or self.target + return buildenvironment.BuildEnvironment(settings, target, **kws) - def test_arch_overridable(self): - buildenv = buildenvironment.BuildEnvironment(self.settings, - arch='noarch') - self.assertEqual(buildenv.arch, 'noarch') - - def test_sets_default_path(self): - self.settings['keep-path'] = False - self.settings['bootstrap'] = False - olddefaultpath = buildenvironment.BuildEnvironment._default_path - buildenvironment.BuildEnvironment._default_path = self.default_path - buildenv = buildenvironment.BuildEnvironment(self.settings) - buildenvironment.BuildEnvironment._default_path = olddefaultpath - self.assertTrue(self.default_path in buildenv.env['PATH']) - - def test_uses_env_path_with_keep_path(self): - self.settings['keep-path'] = True - - old_osenv = buildenvironment.BuildEnvironment._osenv - buildenvironment.BuildEnvironment._osenv = self.fake_env - buildenv = buildenvironment.BuildEnvironment(self.settings) - buildenvironment.BuildEnvironment._osenv = old_osenv - - self.assertEqual(buildenv.env['PATH'], self.fake_env['PATH']) - - def test_uses_env_path_with_bootstrap(self): - self.settings['bootstrap'] = True - - old_osenv = buildenvironment.BuildEnvironment._osenv - buildenvironment.BuildEnvironment._osenv = self.fake_env - buildenv = buildenvironment.BuildEnvironment(self.settings) - buildenvironment.BuildEnvironment._osenv = old_osenv - - self.assertEqual(buildenv.env['PATH'], self.fake_env['PATH']) + def new_build_env(self, settings=None, arch='x86_64'): + settings = settings or self.settings + return buildenvironment.BuildEnvironment(settings, arch) def test_copies_whitelist_vars(self): env = self.fake_env safe = { 'DISTCC_HOSTS': 'example.com:example.co.uk', - 'TMPDIR': '/buildenv/tmp/dir', 'LD_PRELOAD': '/buildenv/lib/libbuildenv.so', 'LD_LIBRARY_PATH': '/buildenv/lib:/buildenv/lib64', 'FAKEROOTKEY': 'b011de73', @@ -88,22 +53,22 @@ class BuildEnvironmentTests(unittest.TestCase): 'FAKEROOT_FD_BASE': '-1', } env.update(safe) - old_osenv = buildenvironment.BuildEnvironment._osenv buildenvironment.BuildEnvironment._osenv = env - buildenv = buildenvironment.BuildEnvironment(self.settings) - buildenvironment.BuildEnvironment._osenv = old_osenv + buildenv = self.new_build_env() self.assertEqual(sorted(safe.items()), sorted([(k, buildenv.env[k]) for k in safe.keys()])) + buildenvironment.BuildEnvironment._osenv = old_osenv + def test_user_spellings_equal(self): - buildenv = buildenvironment.BuildEnvironment(self.settings) + buildenv = self.new_build_env() self.assertTrue(buildenv.env['USER'] == buildenv.env['USERNAME'] == buildenv.env['LOGNAME']) def test_environment_overrides(self): - buildenv = buildenvironment.BuildEnvironment(self.settings) + buildenv = self.new_build_env() self.assertEqual(buildenv.env['TERM'], buildenv._override_term) self.assertEqual(buildenv.env['SHELL'], buildenv._override_shell) self.assertEqual(buildenv.env['USER'], buildenv._override_username) @@ -112,20 +77,36 @@ class BuildEnvironmentTests(unittest.TestCase): self.assertEqual(buildenv.env['LC_ALL'], buildenv._override_locale) self.assertEqual(buildenv.env['HOME'], buildenv._override_home) - def test_environment_settings_set(self): - buildenv = buildenvironment.BuildEnvironment(self.settings) - self.assertEqual(buildenv.env['TOOLCHAIN_TARGET'], - self.settings['toolchain-target']) - self.assertEqual(buildenv.env['CFLAGS'], - self.settings['target-cflags']) - self.assertEqual(buildenv.env['PREFIX'], - self.settings['prefix']) - self.assertEqual(buildenv.env['BOOTSTRAP'], - 'true' if self.settings['bootstrap'] else 'false') + def test_arch_x86_64(self): + b = self.new_build_env(arch='x86_64') + self.assertEqual(b.env['MORPH_ARCH'], 'x86_64') + self.assertEqual(b.env['TARGET'], 'x86_64-baserock-linux-gnu') + self.assertEqual(b.env['TARGET_STAGE1'], 'x86_64-bootstrap-linux-gnu') + + def test_arch_x86_32(self): + b = self.new_build_env(arch='x86_32') + self.assertEqual(b.env['MORPH_ARCH'], 'x86_32') + self.assertEqual(b.env['TARGET'], 'i686-baserock-linux-gnu') + self.assertEqual(b.env['TARGET_STAGE1'], 'i686-bootstrap-linux-gnu') + + def test_arch_armv7l(self): + b = self.new_build_env(arch='armv7l') + self.assertEqual(b.env['MORPH_ARCH'], 'armv7l') + self.assertEqual(b.env['TARGET'], 'armv7l-baserock-linux-gnueabi') + self.assertEqual(b.env['TARGET_STAGE1'], + 'armv7l-bootstrap-linux-gnueabi') + + def test_arch_armv7b(self): + b = self.new_build_env(arch='armv7b') + self.assertEqual(b.env['MORPH_ARCH'], 'armv7b') + self.assertEqual(b.env['TARGET'], 'armv7b-baserock-linux-gnueabi') + self.assertEqual(b.env['TARGET_STAGE1'], + 'armv7b-bootstrap-linux-gnueabi') def test_ccache_vars_set(self): - self.settings['no-ccache'] = False - self.settings['no-distcc'] = False - buildenv = buildenvironment.BuildEnvironment(self.settings) - self.assertTrue(buildenv._ccache_path in buildenv.env['PATH']) + new_settings = copy.copy(self.settings) + new_settings['no-ccache'] = False + new_settings['no-distcc'] = False + buildenv = self.new_build_env(settings=new_settings) + self.assertTrue(buildenv._ccache_path in buildenv.extra_path) self.assertEqual(buildenv.env['CCACHE_PREFIX'], 'distcc') diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 73745d66..f8f4ea88 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -54,6 +54,9 @@ def ldconfig(runcmd, rootdir): # pragma: no cover ''' + # FIXME: use the version in ROOTDIR, since even in + # bootstrap it will now always exist due to being part of build-essential + conf = os.path.join(rootdir, 'etc', 'ld.so.conf') if os.path.exists(conf): logging.debug('Running ldconfig for %s' % rootdir) @@ -152,15 +155,14 @@ class BuilderBase(object): '''Base class for building artifacts.''' def __init__(self, app, staging_area, local_artifact_cache, - remote_artifact_cache, artifact, repo_cache, - build_env, max_jobs, setup_mounts): + remote_artifact_cache, artifact, repo_cache, max_jobs, + setup_mounts): self.app = app self.staging_area = staging_area self.local_artifact_cache = local_artifact_cache self.remote_artifact_cache = remote_artifact_cache self.artifact = artifact self.repo_cache = repo_cache - self.build_env = build_env self.max_jobs = max_jobs self.build_watch = morphlib.stopwatch.Stopwatch() self.setup_mounts = setup_mounts @@ -250,7 +252,6 @@ class BuilderBase(object): return a def runcmd(self, *args, **kwargs): - kwargs['env'] = self.build_env.env return self.staging_area.runcmd(*args, **kwargs) @@ -375,7 +376,7 @@ class ChunkBuilder(BuilderBase): relative_builddir = self.staging_area.relative(builddir) relative_destdir = self.staging_area.relative(destdir) - self.build_env.env['DESTDIR'] = relative_destdir + extra_env = { 'DESTDIR': relative_destdir } steps = [ ('pre-configure', False), @@ -403,9 +404,9 @@ class ChunkBuilder(BuilderBase): max_jobs = self.artifact.source.morphology['max-jobs'] if max_jobs is None: max_jobs = self.max_jobs - self.build_env.env['MAKEFLAGS'] = '-j%s' % max_jobs + extra_env['MAKEFLAGS'] = '-j%s' % max_jobs else: - self.build_env.env['MAKEFLAGS'] = '-j1' + extra_env['MAKEFLAGS'] = '-j1' try: # flushing is needed because writes from python and # writes from being the output in Popen have different @@ -413,6 +414,7 @@ class ChunkBuilder(BuilderBase): logfile.write('# # %s\n' % cmd) logfile.flush() self.runcmd(['sh', '-c', cmd], + extra_env=extra_env, cwd=relative_builddir, stdout=logfile, stderr=subprocess.STDOUT) @@ -459,13 +461,18 @@ class StratumBuilder(BuilderBase): '''Build stratum artifacts.''' + def is_constituent(self, artifact): # pragma: no cover + '''True if artifact should be included in the stratum artifact''' + return (artifact.source.morphology['kind'] == 'chunk' and \ + artifact.source.build_mode != 'bootstrap') + def build_and_cache(self): # pragma: no cover with self.build_watch('overall-build'): - constituents = [dependency - for dependency in self.artifact.dependencies - if dependency.source.morphology['kind'] == 'chunk'] + constituents = [d for d in self.artifact.dependencies + if self.is_constituent(d)] if len(constituents) == 0: logging.warning('Stratum %s is empty' % self.artifact.name) + # the only reason the StratumBuilder has to download chunks is to # check for overlap now that strata are lists of chunks with self.build_watch('check-chunks'): @@ -667,14 +674,12 @@ class Builder(object): # pragma: no cover } def __init__(self, app, staging_area, local_artifact_cache, - remote_artifact_cache, repo_cache, build_env, max_jobs, - setup_mounts): + remote_artifact_cache, repo_cache, max_jobs, setup_mounts): self.app = app self.staging_area = staging_area self.local_artifact_cache = local_artifact_cache self.remote_artifact_cache = remote_artifact_cache self.repo_cache = repo_cache - self.build_env = build_env self.max_jobs = max_jobs self.setup_mounts = setup_mounts @@ -683,8 +688,8 @@ class Builder(object): # pragma: no cover o = self.classes[kind](self.app, self.staging_area, self.local_artifact_cache, self.remote_artifact_cache, artifact, - self.repo_cache, self.build_env, - self.max_jobs, self.setup_mounts) + self.repo_cache, self.max_jobs, + self.setup_mounts) logging.debug('Builder.build: artifact %s with %s' % (artifact.name, repr(o))) built_artifacts = o.build_and_cache() diff --git a/morphlib/builder2_tests.py b/morphlib/builder2_tests.py index df07a59f..c0da3cd9 100644 --- a/morphlib/builder2_tests.py +++ b/morphlib/builder2_tests.py @@ -35,8 +35,9 @@ class FakeApp(object): class FakeStagingArea(object): - def __init__(self, runcmd): + def __init__(self, runcmd, build_env): self.runcmd = runcmd + self.env = build_env.env class FakeSource(object): @@ -146,7 +147,7 @@ class BuilderBaseTests(unittest.TestCase): def setUp(self): self.commands_run = [] self.app = FakeApp(self.fake_runcmd) - self.staging_area = FakeStagingArea(self.fake_runcmd) + self.staging_area = FakeStagingArea(self.fake_runcmd, FakeBuildEnv()) self.artifact_cache = FakeArtifactCache() self.artifact = FakeArtifact('le-artifact') self.repo_cache = None @@ -158,7 +159,6 @@ class BuilderBaseTests(unittest.TestCase): None, self.artifact, self.repo_cache, - self.build_env, self.max_jobs, False) @@ -252,8 +252,7 @@ class ChunkBuilderTests(unittest.TestCase): def setUp(self): self.app = FakeApp() self.build = morphlib.builder2.ChunkBuilder(self.app, None, None, - None, None, None, None, 1, - False) + None, None, None, 1, False) def test_uses_morphology_commands_when_given(self): m = {'build-commands': ['build-it']} diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index a4ea10ed..6acf654b 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -27,9 +27,9 @@ class CacheKeyComputer(object): self._calculated = {} def _filterenv(self, env): - return dict([(k, env[k]) for k in ("USER", "USERNAME", "LOGNAME", - "TOOLCHAIN_TARGET", "PREFIX", - "BOOTSTRAP", "CFLAGS")]) + keys = ["LOGNAME", "MORPH_ARCH", "TARGET", "TARGET_STAGE1", + "USER", "USERNAME"] + return dict([(k, env[k]) for k in keys]) def compute_key(self, artifact): logging.debug('computing cache key for artifact %s from source ' @@ -79,7 +79,6 @@ class CacheKeyComputer(object): def _calculate(self, artifact): keys = { - 'arch': self._build_env.arch, 'env': self._filterenv(self._build_env.env), 'filename': artifact.source.filename, 'kids': [self.compute_key(x) for x in artifact.dependencies] @@ -87,6 +86,8 @@ class CacheKeyComputer(object): kind = artifact.source.morphology['kind'] if kind == 'chunk': + keys['build-mode'] = artifact.source.build_mode + keys['prefix'] = artifact.source.prefix keys['tree'] = artifact.source.tree elif kind in ('system', 'stratum'): morphology = artifact.source.morphology diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py index 411ad3f5..2f033a7a 100644 --- a/morphlib/cachekeycomputer_tests.py +++ b/morphlib/cachekeycomputer_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,6 +14,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import copy import unittest import morphlib @@ -24,8 +25,7 @@ class DummyBuildEnvironment: settings to pick the environment, it just gets passed a dict representing it ''' - def __init__(self, env, arch=None): - self.arch = morphlib.util.arch() if arch is None else arch + def __init__(self, env, arch): self.env = env @@ -103,13 +103,12 @@ class CacheKeyComputerTests(unittest.TestCase): elif m['kind'] == 'chunk': m.builds_artifacts = [m['name']] self.build_env = DummyBuildEnvironment({ - "USER": "foouser", - "USERNAME": "foouser", "LOGNAME": "foouser", - "TOOLCHAIN_TARGET": "dummy-baserock-linux-gnu", - "PREFIX": "/baserock", - "BOOTSTRAP": "false", - "CFLAGS": "-O4"}) + "MORPH_ARCH": "dummy", + "TARGET": "dummy-baserock-linux-gnu", + "TARGET_STAGE1": "dummy-baserock-linux-gnu", + "USER": "foouser", + "USERNAME": "foouser"}, 'dummy') self.artifact_resolver = morphlib.artifactresolver.ArtifactResolver() self.artifacts = self.artifact_resolver.resolve_artifacts( self.source_pool) @@ -155,14 +154,8 @@ class CacheKeyComputerTests(unittest.TestCase): def test_different_env_gives_different_key(self): artifact = self._find_artifact('system-rootfs') oldsha = self.ckc.compute_key(artifact) - build_env = DummyBuildEnvironment({ - "USER": "foouser", - "USERNAME": "foouser", - "LOGNAME": "foouser", - "TOOLCHAIN_TARGET": "dummy-baserock-linux-gnu", - "PREFIX": "/baserock", - "BOOTSTRAP": "false", - "CFLAGS": "-Os"}) + build_env = copy.deepcopy(self.build_env) + build_env.env["USER"] = "brian" ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) self.assertNotEqual(oldsha, ckc.compute_key(artifact)) diff --git a/morphlib/exts/kvm.write b/morphlib/exts/kvm.write index 1579dc1f..c491f206 100755 --- a/morphlib/exts/kvm.write +++ b/morphlib/exts/kvm.write @@ -99,11 +99,18 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): '''Create the libvirt virtual machine.''' self.status(msg='Creating libvirt/kvm virtual machine') + + attach_disks = self.parse_attach_disks() + attach_opts = [] + for disk in attach_disks: + attach_opts.extend(['--disk', 'path=%s' % disk]) + cliapp.runcmd( ['ssh', ssh_host, 'virt-install', '--connect qemu:///system', '--import', '--name', vm_name, '--ram', '1024', '--vnc', '--noreboot', - '--disk path=%s,bus=ide' % vm_path]) + '--disk path=%s,bus=ide' % vm_path] + + attach_opts) KvmPlusSshWriteExtension().run() diff --git a/morphlib/exts/virtualbox-ssh.write b/morphlib/exts/virtualbox-ssh.write index 9b99c7a1..862d4f02 100755 --- a/morphlib/exts/virtualbox-ssh.write +++ b/morphlib/exts/virtualbox-ssh.write @@ -121,7 +121,17 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): '--port', '0', '--device', '0', '--type', 'hdd', '--medium', vdi_path], ] - + + attach_disks = self.parse_attach_disks() + for device_no, disk in enumerate(attach_disks, 1): + cmd = ['storageattach', vm_name, + '--storagectl', '"SATA Controller"', + '--port', str(device_no), + '--device', '0', + '--type', 'hdd', + '--medium', disk] + commands.append(cmd) + for command in commands: argv = ['ssh', ssh_host, 'VBoxManage'] + command cliapp.runcmd(argv) diff --git a/morphlib/git.py b/morphlib/git.py index c491991c..9cdd0a84 100644 --- a/morphlib/git.py +++ b/morphlib/git.py @@ -80,7 +80,7 @@ class Submodules(object): # try to read the .gitmodules file from the repo/ref content = self.app.runcmd( ['git', 'cat-file', 'blob', '%s:.gitmodules' % self.ref], - cwd=self.repo) + cwd=self.repo, ignore_fail=True) # drop indentation in sections, as RawConfigParser cannot handle it return '\n'.join([line.strip() for line in content.splitlines()]) diff --git a/morphlib/morph2.py b/morphlib/morph2.py index 3cdf49a9..a8e1d7d3 100644 --- a/morphlib/morph2.py +++ b/morphlib/morph2.py @@ -52,7 +52,7 @@ class Morphology(object): 'stratum': [ ('chunks', []), ('description', ''), - ('build-depends', None) + ('build-depends', None), ], 'system': [ ('strata', []), @@ -157,6 +157,10 @@ class Morphology(object): self._set_default_value(source, 'morph', source['name']) if 'build-depends' not in source: self._set_default_value(source, 'build-depends', None) + if 'build-mode' not in source: + self._set_default_value(source, 'build-mode', 'staging') + if 'prefix' not in source: + self._set_default_value(source, 'prefix', '/usr') def _parse_size(self, size): if isinstance(size, basestring): diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py index 817d7fcd..5b68f1a8 100644 --- a/morphlib/morphologyfactory.py +++ b/morphlib/morphologyfactory.py @@ -106,6 +106,16 @@ class MorphologyFactory(object): '(arch is a mandatory field)' % filename) + valid_archs = ['armv7l', 'armv7b', 'x86_32', 'x86_64'] + + if morphology['arch'] == 'armv7': + morphology._dict['arch'] = 'armv7l' + + if morphology['arch'] not in valid_archs: + raise morphlib.Error('Unknown arch %s. This version of Morph ' + 'supports the following architectures: %s' % + (morphology['arch'], ', '.join(valid_archs))) + if not morphology['system-kind']: raise morphlib.Error('No system-kind defined in system %s ' '(it is a mandatory field)' % filename) @@ -124,7 +134,6 @@ class MorphologyFactory(object): name = morphology['name'] morphology.builds_artifacts = [name + '-rootfs'] - morphology.needs_staging_area = False morphology.needs_artifact_metadata_cached = False def _check_and_tweak_stratum(self, morphology, reponame, sha1, filename): @@ -140,7 +149,6 @@ class MorphologyFactory(object): (filename, name)) morphology.builds_artifacts = [morphology['name']] - morphology.needs_staging_area = False morphology.needs_artifact_metadata_cached = True def _check_and_tweak_chunk(self, morphology, reponame, sha1, filename): @@ -151,5 +159,4 @@ class MorphologyFactory(object): else: morphology.builds_artifacts = [morphology['name']] - morphology.needs_staging_area = True morphology.needs_artifact_metadata_cached = False diff --git a/morphlib/morphologyfactory_tests.py b/morphlib/morphologyfactory_tests.py index dbdb4228..dd454a30 100644 --- a/morphlib/morphologyfactory_tests.py +++ b/morphlib/morphologyfactory_tests.py @@ -73,7 +73,7 @@ class FakeLocalRepo(object): } def __init__(self): - self.arch = 'unknown' + self.arch = 'x86_64' self.system_kind = 'unknown' def cat(self, sha1, filename): @@ -225,22 +225,10 @@ class MorphologyFactoryTests(unittest.TestCase): self.assertEqual(morph.builds_artifacts, ['stratum']) def test_sets_build_artifacts_for_system(self): - self.lr.arch = 'x86_64' + self.lr.arch = 'x86_32' morph = self.mf.get_morphology('reponame', 'sha1', 'system.morph') self.assertEqual(morph.builds_artifacts, ['system-rootfs']) - def test_sets_needs_staging_for_chunk(self): - morph = self.mf.get_morphology('reponame', 'sha1', 'chunk.morph') - self.assertEqual(morph.needs_staging_area, True) - - def test_does_not_set_needs_staging_for_stratum(self): - morph = self.mf.get_morphology('reponame', 'sha1', 'stratum.morph') - self.assertEqual(morph.needs_staging_area, False) - - def test_does_not_set_needs_staging_for_system(self): - morph = self.mf.get_morphology('reponame', 'sha1', 'system.morph') - self.assertEqual(morph.needs_staging_area, False) - def test_does_not_set_needs_artifact_metadata_cached_for_chunk(self): morph = self.mf.get_morphology('reponame', 'sha1', 'chunk.morph') self.assertEqual(morph.needs_artifact_metadata_cached, False) @@ -253,6 +241,17 @@ class MorphologyFactoryTests(unittest.TestCase): morph = self.mf.get_morphology('reponame', 'sha1', 'system.morph') self.assertEqual(morph.needs_artifact_metadata_cached, False) + + def test_arch_is_validated(self): + self.lr.arch = 'unknown' + self.assertRaises(morphlib.Error, self.mf.get_morphology, + 'reponame', 'sha1', 'system.morph') + + def test_arch_arm_defaults_to_le(self): + self.lr.arch = 'armv7' + morph = self.mf.get_morphology('reponame', 'sha1', 'system.morph') + self.assertEqual(morph['arch'], 'armv7l') + def test_fails_if_system_does_not_define_system_kind(self): self.lr.system_kind = '' self.assertRaises(morphlib.Error, self.mf.get_morphology, diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index 141c3187..32158a88 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -73,7 +73,14 @@ class BranchAndMergePlugin(cliapp.Plugin): arg_synopsis='SYSTEM') self.app.add_subcommand('status', self.status) self.app.add_subcommand('branch-from-image', self.branch_from_image, - arg_synopsis='REPO BRANCH [METADATADIR]') + arg_synopsis='REPO BRANCH') + group_branch = 'Branching Options' + self.app.settings.string(['metadata-dir'], + 'Set metadata location for branch-from-image' + ' (default: /baserock)', + metavar='DIR', + default='/baserock', + group=group_branch) # Advanced commands self.app.add_subcommand('foreach', self.foreach, @@ -859,7 +866,7 @@ class BranchAndMergePlugin(cliapp.Plugin): 'branch-from-image needs repository, ref and path to metadata') root_repo = args[0] branch = args[1] - metadata_path = '/baserock' if len(args) == 2 else args[2] + metadata_path = self.app.settings['metadata-dir'] workspace = self.deduce_workspace() self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) diff --git a/morphlib/plugins/syslinux-disk-systembuilder_plugin.py b/morphlib/plugins/syslinux-disk-systembuilder_plugin.py index cc47389c..f05375f2 100644 --- a/morphlib/plugins/syslinux-disk-systembuilder_plugin.py +++ b/morphlib/plugins/syslinux-disk-systembuilder_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -84,16 +84,13 @@ class SyslinuxDiskBuilder(DiskImageBuilder): # pragma: no cover class SyslinuxDiskBuilderPlugin(cliapp.Plugin): def enable(self): - # Only provide this system builder on architectures that are - # supported by syslinux. - if morphlib.util.arch() in ['x86_64', 'i386', 'i486', 'i586', 'i686']: - self.app.settings.string_list( - ['syslinux-mbr-search-paths'], - 'A list of files to search for to use as the syslinux mbr', - default=['/usr/lib/extlinux/mbr.bin', - '/usr/share/syslinux/mbr.bin'], - group='Build Options') - self.app.system_kind_builder_factory.register(SyslinuxDiskBuilder) + self.app.settings.string_list( + ['syslinux-mbr-search-paths'], + 'A list of files to search for to use as the syslinux mbr', + default=['/usr/lib/extlinux/mbr.bin', + '/usr/share/syslinux/mbr.bin'], + group='Build Options') + self.app.system_kind_builder_factory.register(SyslinuxDiskBuilder) def disable(self): pass diff --git a/morphlib/plugins/trebuchet_plugin.py b/morphlib/plugins/trebuchet_plugin.py index 1ebffbf4..036fec50 100644 --- a/morphlib/plugins/trebuchet_plugin.py +++ b/morphlib/plugins/trebuchet_plugin.py @@ -46,19 +46,32 @@ class TrebuchetPlugin(cliapp.Plugin): repo_name2, ref2, filename2 = args[4:7] app = self.app - build_env = morphlib.buildenvironment.BuildEnvironment(app.settings) - ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) lac, rac = morphlib.util.new_artifact_caches(app.settings) lrc, rrc = morphlib.util.new_repo_caches(app) + def get_system_source(repo_name, ref, filename): + srcpool = app.create_source_pool( + lrc, rrc, (repo_name, ref, filename)) + src = srcpool.lookup(repo_name, ref, filename) + return srcpool, src.morphology['arch'] + + srcpool1, arch1 = get_system_source(repo_name1, ref1, filename1) + srcpool2, arch2 = get_system_source(repo_name2, ref2, filename2) + + if arch1 != arch2: + raise cliapp.AppException('System architectures do not match: ' + '%s vs. %s' % (arch1, arch2)) + + build_env = morphlib.buildenvironment.BuildEnvironment( + app.settings, arch1) + ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) + def the_one(source, repo_name, ref, filename): return (source.repo_name == repo_name and source.original_ref == ref and source.filename == filename) - def get_artifact(repo_name, ref, filename): - srcpool = app.create_source_pool( - lrc, rrc, (repo_name, ref, filename)) + def get_artifact(srcpool, repo_name, ref, filename): ar = morphlib.artifactresolver.ArtifactResolver() artifacts = ar.resolve_artifacts(srcpool) for artifact in artifacts: @@ -70,8 +83,8 @@ class TrebuchetPlugin(cliapp.Plugin): a.cache_key = artifact.cache_key return a - artifact1 = get_artifact(repo_name1, ref1, filename1) - artifact2 = get_artifact(repo_name2, ref2, filename2) + artifact1 = get_artifact(srcpool1, repo_name1, ref1, filename1) + artifact2 = get_artifact(srcpool2, repo_name2, ref2, filename2) image_path_1 = lac.get(artifact1).name image_path_2 = lac.get(artifact2).name diff --git a/morphlib/source.py b/morphlib/source.py index d4f1e119..99b0a993 100644 --- a/morphlib/source.py +++ b/morphlib/source.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -30,11 +30,6 @@ class Source(object): * ``tree`` -- the SHA1 of the tree corresponding to the commit * ``morphology`` -- the in-memory representation of the morphology we use * ``filename`` -- basename of the morphology filename - * ``dependencies`` -- list of Sources for build dependencies for us - * ``dependents`` -- list of Source for whom we are a build dependency - - The ``dependencies`` and ``dependents`` lists MUST be modified by - the ``add_dependencies`` and ``add_dependent`` methods only. ''' diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index ae9e7e39..8150ba42 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012,2013 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,6 +18,7 @@ import logging import os import shutil import stat +import cliapp from urlparse import urlparse import morphlib @@ -27,32 +28,43 @@ class StagingArea(object): '''Represent the staging area for building software. - The build dependencies of what will be built will be installed in the - staging area. The staging area may be a dedicated part of the - filesystem, used with chroot, or it can be the actual root of the - filesystem, which is needed when bootstrap building Baserock. The - caller chooses this by providing the root directory of the staging - area when the object is created. The directory must already exist. - - The staging area can also install build artifacts. + The staging area is a temporary directory. In normal operation the build + dependencies of the artifact being built are installed into the staging + area and then 'chroot' is used to isolate the build processes from the host + system. Chunks built in 'test' or 'build-essential' mode have an empty + staging area and are allowed to use the tools of the host. ''' - def __init__(self, app, dirname, tempdir): + _base_path = ['/sbin', '/usr/sbin', '/bin', '/usr/bin'] + + def __init__(self, app, dirname, build_env, use_chroot=True, extra_env={}, + extra_path=[]): self._app = app self.dirname = dirname - self.tempdir = tempdir self.builddirname = None self.destdirname = None self.mounted = None self._bind_readonly_mount = None + self.use_chroot = use_chroot + self.env = build_env.env + self.env.update(extra_env) + + if use_chroot: + path = extra_path + build_env.extra_path + self._base_path + else: + rel_path = extra_path + build_env.extra_path + full_path = [os.path.normpath(dirname + p) for p in rel_path] + path = full_path + os.environ['PATH'].split(':') + self.env['PATH'] = ':'.join(path) + # Wrapper to be overridden by unit tests. def _mkdir(self, dirname): # pragma: no cover os.mkdir(dirname) def _dir_for_source(self, source, suffix): - dirname = os.path.join(self.tempdir, + dirname = os.path.join(self.dirname, '%s.%s' % (source.morphology['name'], suffix)) self._mkdir(dirname) return dirname @@ -79,6 +91,9 @@ class StagingArea(object): def relative(self, filename): '''Return a filename relative to the staging area.''' + if not self.use_chroot: + return filename + dirname = self.dirname if not dirname.endswith('/'): dirname += '/' @@ -194,8 +209,7 @@ class StagingArea(object): if not os.path.isdir(ccache_repodir): os.mkdir(ccache_repodir) # Get the destination path - ccache_destdir= os.path.join(self.tempdir, - 'tmp', 'ccache') + ccache_destdir= os.path.join(self.dirname, 'tmp', 'ccache') # Make sure that the destination exists. We'll create /tmp if necessary # to avoid breaking when faced with an empty staging area. if not os.path.isdir(ccache_destdir): @@ -251,30 +265,39 @@ class StagingArea(object): def runcmd(self, argv, **kwargs): # pragma: no cover '''Run a command in a chroot in the staging area.''' - - cwd = kwargs.get('cwd') or '/' - if 'cwd' in kwargs: - cwd = kwargs['cwd'] - del kwargs['cwd'] - else: - cwd = '/' - if self._app.settings['staging-chroot']: - not_readonly_dirs = [self.builddirname, self.destdirname, + assert 'env' not in kwargs + kwargs['env'] = self.env + if 'extra_env' in kwargs: + kwargs['env'].update(kwargs['extra_env']) + del kwargs['extra_env'] + + if self.use_chroot: + cwd = kwargs.get('cwd') or '/' + if 'cwd' in kwargs: + cwd = kwargs['cwd'] + del kwargs['cwd'] + else: + cwd = '/' + + do_not_mount_dirs = [self.builddirname, self.destdirname, 'dev', 'proc', 'tmp'] - dirs = os.listdir(self.dirname) - for excluded_dir in not_readonly_dirs: - dirs.remove(excluded_dir) real_argv = ['linux-user-chroot'] - - for entry in dirs: - real_argv += ['--mount-readonly', '/'+entry] - + for d in os.listdir(self.dirname): + if d not in do_not_mount_dirs: + if os.path.isdir(os.path.join(self.dirname, d)): + real_argv += ['--mount-readonly', '/'+d] real_argv += [self.dirname] - else: - real_argv = ['chroot', '/'] - real_argv += ['sh', '-c', 'cd "$1" && shift && exec "$@"', '--', cwd] - real_argv += argv + real_argv += ['sh', '-c', 'cd "$1" && shift && exec "$@"', '--', + cwd] + real_argv += argv - return self._app.runcmd(real_argv, **kwargs) + try: + return self._app.runcmd(real_argv, **kwargs) + except cliapp.AppException as e: + raise cliapp.AppException('In staging area %s: running ' + 'command \'%s\' failed.' % + (self.dirname, ' '.join(argv))) + else: + return self._app.runcmd(argv, **kwargs) diff --git a/morphlib/stagingarea_tests.py b/morphlib/stagingarea_tests.py index 313226d2..35174f3b 100644 --- a/morphlib/stagingarea_tests.py +++ b/morphlib/stagingarea_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012,2013 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,6 +24,13 @@ import unittest import morphlib +class FakeBuildEnvironment(object): + + def __init__(self): + self.env = { + } + self.extra_path = ['/extra-path'] + class FakeSource(object): def __init__(self): @@ -56,9 +63,10 @@ class StagingAreaTests(unittest.TestCase): os.mkdir(os.path.join(self.cachedir, 'artifacts')) self.staging = os.path.join(self.tempdir, 'staging') self.created_dirs = [] + self.build_env = FakeBuildEnvironment() self.sa = morphlib.stagingarea.StagingArea( - FakeApplication(self.cachedir, self.tempdir), - self.staging, self.staging) + FakeApplication(self.cachedir, self.tempdir), self.staging, + self.build_env) def tearDown(self): shutil.rmtree(self.tempdir) @@ -89,10 +97,6 @@ class StagingAreaTests(unittest.TestCase): def test_remembers_specified_directory(self): self.assertEqual(self.sa.dirname, self.staging) - def test_accepts_root_directory(self): - sa = morphlib.stagingarea.StagingArea(object(), '/', '/tmp') - self.assertEqual(sa.dirname, '/') - def test_creates_build_directory(self): source = FakeSource() self.sa._mkdir = self.fake_mkdir @@ -123,3 +127,9 @@ class StagingAreaTests(unittest.TestCase): self.sa.install_artifact(f) self.sa.remove() self.assertFalse(os.path.exists(self.staging)) + + def test_supports_non_isolated_mode(self): + sa = morphlib.stagingarea.StagingArea( + object(), self.staging, self.build_env, use_chroot=False) + filename = os.path.join(self.staging, 'foobar') + self.assertEqual(sa.relative(filename), filename) diff --git a/morphlib/util.py b/morphlib/util.py index b4e06092..c77ac8d3 100644 --- a/morphlib/util.py +++ b/morphlib/util.py @@ -38,11 +38,6 @@ except NotImplementedError: # pragma: no cover import os -def arch(): - '''Return the CPU architecture of the current host.''' - return os.uname()[4] - - def indent(string, spaces=4): '''Return ``string`` indented by ``spaces`` spaces. diff --git a/morphlib/util_tests.py b/morphlib/util_tests.py index 4f472d4e..89fe184e 100644 --- a/morphlib/util_tests.py +++ b/morphlib/util_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2012 Codethink Limited +# Copyright (C) 2011-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,14 +19,6 @@ import unittest import morphlib -class ArchTests(unittest.TestCase): - - def test(self): - arch = morphlib.util.arch() - self.assertEqual(type(arch), str) - self.assertNotEqual(arch, '') - - class IndentTests(unittest.TestCase): def test_returns_empty_string_for_empty_string(self): diff --git a/morphlib/writeexts.py b/morphlib/writeexts.py index a479f3cb..469b8557 100755 --- a/morphlib/writeexts.py +++ b/morphlib/writeexts.py @@ -184,3 +184,11 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['sync']) time.sleep(2) + def parse_attach_disks(self): + '''Parse $ATTACH_DISKS into list of disks to attach.''' + + if 'ATTACH_DISKS' in os.environ: + s = os.environ['ATTACH_DISKS'] + return s.split(':') + else: + return [] |